From abf2c1d2c2750c18eef21a10fa7c8106e09055d5 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Mon, 21 Jun 2021 21:36:15 +0530 Subject: [PATCH 01/16] Fixed S3 file copy defect --- charts/oes/Chart.yaml | 2 +- .../spinnaker/templates/configmap/halyard-init-script.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/oes/Chart.yaml b/charts/oes/Chart.yaml index 6b42813b..f80549e7 100644 --- a/charts/oes/Chart.yaml +++ b/charts/oes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: oes -version: 3.8.0 +version: 3.8.1 appVersion: 3.8.0 description: OES is a non-forked version of OSS spinnaker icon: https://www.opsmx.com/images/logo.png diff --git a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml index 1f5a2276..cdcae3ce 100755 --- a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml +++ b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml @@ -89,9 +89,9 @@ data: fi # Copy S3 Specific files cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml - cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml - cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-rw.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml - cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro-deck.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml + cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro.yml + cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-rw.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-rw.yml + cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro-deck.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro-deck.yml cp /tmp/spinnaker/.hal/default/profiles/spinnakerconfig.yml.s3 /tmp/spinnaker/.hal/default/profiles/spinnakerconfig.yml # Edit and put secrets in there sed -i s/AWS_ACCESS_KEY_ID/${S3_ACCESSKEY}/g /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml From 109f38ce54f837c2d31d52c11b474023301f329e Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 23 Jun 2021 21:39:58 +0530 Subject: [PATCH 02/16] Ensured that oes-config runs last --- charts/oes/templates/configmaps/datasource-creation.yaml | 4 ++-- charts/oes/templates/hooks/oes-config-job.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/oes/templates/configmaps/datasource-creation.yaml b/charts/oes/templates/configmaps/datasource-creation.yaml index 10b30933..4f3187c2 100644 --- a/charts/oes/templates/configmaps/datasource-creation.yaml +++ b/charts/oes/templates/configmaps/datasource-creation.yaml @@ -35,8 +35,8 @@ data: if [ "$DECK" == "true" ] && [ "$CLOUDCACHING" == "true" ] && [ "$CLOUDRO" == "true" ] && [ "$CLOUDRW" == "true" ] && [ "$CLOUDRODECK" == "true" ] && [ "$FRONT" == "true" ] && [ "$GATE" == "true" ] && [ "$IGOR" == "true" ] && [ "$ORCA" == "true" ] && [ "$ROSCO" == "true" ] && [ "$ECHOWORKER" == "true" ] && [ "$ECHOSCHEDULER" == "true" ] && [ "$SAPOR" == "true" ] && [ "$PLATFORM" == "true" ] && [ "$AUTOPILOT" == "true" ]; then - echo \"Spinnaker and OES services are Up and Ready, sleeping for min..\" - sleep 2m + echo \"Spinnaker and OES services are Up and Ready..\" + sleep 5 curl --header "Content-Type: application/json" --header "x-spinnaker-user: admin" --request POST --data '{"datasourceType": "OPA", "name": "OPA", "configurationFields": {"endPoint": "opa:8181"}}' http://oes-platform:8095/platformservice/v2/datasources curl --header "Content-Type: application/json" --header "x-spinnaker-user: admin" --request POST --data '{"datasourceType": "AUTOPILOT", "name": "Autopilot", "configurationFields": {"username": "admin"} }' http://oes-platform:8095/platformservice/v2/datasources #curl --header "Content-Type: application/json" --header "x-spinnaker-user: admin" --request POST --data '{"datasourceType": "ELASTICSEARCH", "name": "elastic-default", "configurationFields": {"endPoint": "https://newoeselastic.opsmx.com", "username": "opsmxuser", "password": "OpsMx@123", "kibanaEndPoint": "https://newoeskibana.opsmx.com", "kibanaPassword": "OpsMx@123", "kibanaUsername": "opsmxuser" }}' http://oes-platform:8095/platformservice/v2/datasources diff --git a/charts/oes/templates/hooks/oes-config-job.yaml b/charts/oes/templates/hooks/oes-config-job.yaml index 72c22576..e8fe3f33 100644 --- a/charts/oes/templates/hooks/oes-config-job.yaml +++ b/charts/oes/templates/hooks/oes-config-job.yaml @@ -3,9 +3,9 @@ apiVersion: batch/v1 kind: Job metadata: annotations: - "helm.sh/hook": "post-install" + "helm.sh/hook": "post-install,post-upgrade" "helm.sh/hook-delete-policy": "before-hook-creation" - "helm.sh/hook-weight": "0" + "helm.sh/hook-weight": "5" labels: app: oes component: oes-config From 5b0817c70d837add2ccffb26fc9abd31dd2bc140 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 17:53:47 +0530 Subject: [PATCH 03/16] Changes for pipeline promotion --- .../configmap/halyard-init-script.yaml | 2 + .../pipeline-promotion/git-token-secret.yaml | 4 +- .../local-spin-cli-config-secret.yaml | 2 +- .../pipe-promot-config-cm.yaml | 48 +- .../pipe-promot-scripts-cm.yaml | 1043 +++++++++-------- .../spin-cli-config-secret.yaml | 2 +- 6 files changed, 583 insertions(+), 518 deletions(-) diff --git a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml index cdcae3ce..605b5488 100755 --- a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml +++ b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml @@ -68,6 +68,7 @@ data: if [ -f /tmp/spinnaker/.hal/default/profiles/fiat-local.yml ]; then sed -i s/RELEASE_NAME/{{ .Release.Name }}/g /tmp/spinnaker/.hal/default/profiles/fiat-local.yml fi + sed -i s/SPINNAKER_NAMESPACE/${SPINNAKER_NAMESPACE}/ /tmp/spinnaker/.hal/default/profiles/orca-local.yml printf 'server.address: 0.0.0.0\n' > /tmp/config/halyard-local.yml if [ -f /tmp/spinnaker/.hal/halyard.yaml ]; then cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config @@ -87,6 +88,7 @@ data: if [ -f /tmp/spinnaker/.hal/default/profiles/fiat-local.yml ]; then sed -i s/RELEASE_NAME/{{ .Release.Name }}/g /tmp/spinnaker/.hal/default/profiles/fiat-local.yml fi + sed -i s/SPINNAKER_NAMESPACE/${SPINNAKER_NAMESPACE}/ /tmp/spinnaker/.hal/default/profiles/orca-local.yml # Copy S3 Specific files cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-caching.yml cp /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro.yml.s3 /tmp/spinnaker/.hal/default/service-settings/clouddriver-ro.yml diff --git a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml index 4dcd2d91..14f883f1 100644 --- a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml @@ -3,9 +3,11 @@ apiVersion: v1 stringData: # Git token to access repo where pipeline stuff is stored git_secret_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.token }} + git_pr_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approver_token }} + #git_secret_sshkey: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} kind: Secret metadata: - namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} + #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} name: git-token type: Opaque {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml index 6c2cc22a..d4fe9da1 100644 --- a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml @@ -9,5 +9,5 @@ stringData: kind: Secret metadata: name: local-spin-cli-config - namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} + #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml b/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml index da2895ee..124c3e02 100644 --- a/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml +++ b/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml @@ -1,18 +1,40 @@ -{{- if .Values.spinnaker.gitopsHalyard.pipelinePromotion.enabled }} apiVersion: v1 data: - git_friendly_username: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.username }} - git_project: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.repository }} - git_refresh_enabled: "true" - git_repo: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.organization }} - git_user_email: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.usermail }} - ignore_spin_errors: "false" - debug: "false" - pipelineconfig: "false" # Enable pipeline configuration during promotion - pipelineconfigdir: "prod-config" # directory to look for under application when syncing to Spinnaker - pipelinecreateconf: "false" # Create default parameter-config files in "default-config", useful for 1st time param-creation + repo.properties: | + #properties file for pipeline promotion scripts + + # Common Stuff + repo_type={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.type }} + repo_name={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.repository }} + root_folder={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.rootFolder }} + #S3 Specific + AWS_ACCESS_KEY_ID={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.AWS_SECRET_ACCESS_KEY }} + + #git mandatory patameters + git_url={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.baseUrl }} + git_project={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.organization }} + git_user={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.username }} + git_branch={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.branch }} + #git_password={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.password }} + #API + git_api_url={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.apiUrl }} # bitbucket + + #Auto PR requirements + merge_branch={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.createPR }} + auto_merge={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.autoApprovePR }} + git_approve_user={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approvingUser }} + target_branch={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.targetBranch }} + + #optional + #git_user_email={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.usermail }} + + #delete pipeLine + delete_on_sync_spin={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.deleteOnSyncSpin }} + delete_on_sync_repo={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.deleteOnSyncRepo }} + #git_approve_user_password={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approver_password }} + #git_secret_sshkey={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} kind: ConfigMap metadata: + creationTimestamp: null name: pipe-promot-config - namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} -{{- end }} diff --git a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml index 94bc7f47..0dfa2032 100644 --- a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml +++ b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml @@ -1,535 +1,574 @@ -{{- if .Values.spinnaker.gitopsHalyard.pipelinePromotion.enabled }} apiVersion: v1 -kind: ConfigMap -metadata: - name: pipe-promot-scripts - namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} data: - deployer.sh: | + bitbucket.sh: "#!/bin/bash\n\n#this script funtions only work for bitbucket central + repository\n\n#this script funtions only work for git repo\n#env variables needed + for this to work are as below\n#***git_url=\"example.bitbucket.com\" make sure + you dont add http/https or / in the url\n#****git_repo=\"pipelinepromotion\" repo + to be pushed/download pipeline json files from\n#***git_project=\"kes\" project + key is needed to clone/push/pull merge code\n#***git_user=\"tes.user\" user is + needed for cloning and pusing changes (stash does not support only access key)\n#***git_branch=\"testbranch\" + the branch to which the code should be merged with\n#***merge_branch=false if + true then provide all the below env variables\n# git_secret_token=\"dafjaljoahfoasjoijso\" + needed to create pull requests should be the git_users secret token\n# git_pr_token=\"slkdfjaljoajfopaj\" + this is approver token to approve pull requests / you can also provide approver + password here.\n# git_approve_user=\"test.approver\" username of the pull request + approver\n# git_password=\"adjoowddaw\" make sure your password does not include + special characters like # @*/. special characters cause git clone command to fail + with https\n#\n# repo_type=\"stash\" for selfhosted bitbucket server please use + stash as repo type\n#***root_folder=\"path/to/pipeline-promotion/folder\" folder + to be selected in the repo to which the pipeline jobs to be pushed\n#***command=upload + for running specific job -\n# upload - + to upload spinnaker pipeline json files to repo\n# download + - to download pipeline json file from repo and apply on spinnaker\n#***spinnaker_applications=\"testapp,sampleapp\" + application needed to collect the pipeline information\n#spinnaker_pipelines=\"\" + provide pipelines to be collected, if nothing given, all the pipelines of the + application are collected\n#git_secret_sshkey=\"sshkey\" ssh key if you want to + clone repo with ssh protocol\n\n# note *** env variables are mandatory to work + with the script\n\nsource scripts/git.sh\ngit_bitbucket_api=$git_api_url\npr_id=0\napprove_pr_bitbucket(){\n + \ approve_req=$(curl -X POST -u $git_approve_user:$git_pr_token \\\n $git_bitbucket_api${git_project}/${git_repo}/pullrequests/${pr_id}/approve + -o -I -L -s -w \"%{http_code}\")\n echo $approve_req\n if [[ $approve_req == + \"200\" ]];then\n echo \"merge request approved successfully\"\n else\n echo + \"FAIL: failed to approve the request \"\n exit 1\n fi\n}\n\nmerge_pr_bitbucket(){\n\n + \ merge_req=$(curl -X POST -u $git_user:$git_secret_token \\\n $git_bitbucket_api${git_project}/${git_repo}/pullrequests/${pr_id}/merge + -o -I -L -s -w \"%{http_code}\")\n echo $merge_req\n if [[ $merge_req == 200 + \ ]]; then\n echo \"merged pr successfully\"\n elif [[ $merge_req == 202 ]]; + then\n echo \"merging is in progress will be merged in less than a min\"\n + \ else\n echo \"FAILED: failed to merge $merge_pr\"\n exit 1\nfi\n}\ncreate_pr_bitbucket(){\n\n\tlocal + output=$(curl -X POST -H \"Content-Type: application/json\" -u $git_user:$git_secret_token + \ $git_bitbucket_api${git_project}/${git_repo}/pullrequests -d '{\n \"title\": + \"merging '$git_branch' to '$target_branch'\",\n \"source\": {\n \"branch\": + {\n \"name\": \"'$git_branch'\"\n }\n },\n \"destination\": + {\n \"branch\": {\n \"name\": \"'$target_branch'\"\n + \ }\n }\n}')\n echo $output\n echo $output > pr_response.json\n + \ grep \"There are no changes to be pulled\" pr_response.json\n if [ \"$?\" + = 0 ]\n then\n echo \"master branch is already up-to-date\"\n exit 0\n + \ else\n pr_id=$(cat pr_response.json| jq '(.id)' | sed 's/\\\"//g')\n if + [ $? = 0 ]; then\n echo \"successfully created pull request \"\n #rm + -f pr_response.json\n else\n echo \"ERROR: failed to raise pull request + $output\"\n exit 1\n fi\nfi\n}\n\nsync_spin_to_bitbucket(){\n setup_git\n + \ sync_spin_to_git\n if [[ $merge_branch == \"true\" && $target_branch != \"\" + && ($git_branch != $target_branch) ]];then\n if [[ $git_api_url_port != \"\" + ]];then\n git_bitbucket_api=$git_bitbucket_api:$git_api_url_port\n create_pr_bitbucket\n + \ if [[ $auto_merge == \"true\" ]]; then\n approve_pr_bitbucket\n sleep + 5\n merge_pr_bitbucket\n fi\n else\n create_pr_bitbucket\n + \ if [[ $auto_merge == \"true\" ]]; then\n approve_pr_bitbucket\n sleep + 5\n merge_pr_bitbucket\n fi\n fi\n fi\n}\n" + deployer.sh: "#!/bin/bash\necho \"In deployer.sh\"\nSBASE=scripts\nsource config/repo.properties\nsource + $SBASE/spin.sh\nsource $SBASE/stash.sh\nsource $SBASE/s3.sh\nsource $SBASE/github.sh\nsource + $SBASE/bitbucket.sh\necho \"Sourcing complete\"\nsync_repo_from_spinnaker(){\n\tif + [[ $repo_type = \"s3\" ]];\n \tthen\n\t upload_spin_to_s3\n\telif [[ $repo_type + = \"stash\" ]]; then\n\t\tsync_spin_to_stash\n elif [[ $repo_type == \"bitbucket\" + ]]; then\n\t\tsync_spin_to_bitbucket\n\telif [[ $repo_type = \"git\" ]]; then\n\t\tsync_spin_to_github\n\tfi\n}\nsync_spinnaker_from_repo(){\n\tif + [[ $repo_type = \"s3\" ]];\n \tthen\n\t sync_from_s3_spin\n\telif [[ $repo_type + = \"stash\" || $repo_type = \"git\" || $repo_type = \"bitbucket\" ]]; then\n\t\tsync_git_to_spin\n\tfi\n}\n\nif + [[ \"$command\" == \"download\" ]]; then\n\tsync_spinnaker_from_repo\nelif [[ + \"$command\" == \"upload\" ]]; then\n echo \"executing upload\"\n\t#statement\n\tsync_repo_from_spinnaker\nelse\n\techo + \"ERROR: unknown command\"\n\nfi\necho \"Done executing\"\n" + git.sh: "#!/bin/bash\nsource scripts/spin.sh\n\ngit_repo=$repo_name\ntempdir=\"/tmp/\"\npull_requred=false\nsetup_git() + {\n echo \"Setting up the Git \"\n local name=${git_user:-spinnaker}\n local + email=${git_user_email:-spinnaker@symphony.com}\n git config --global user.email + \"$email\"\n git config --global user.name \"$name\"\n}\ngit_clone_http() {\n + \ echo \"cloning $git_project/$git_repo over https\"\n if [[ $repo_type == \"git\" + || $repo_type == \"bitbucket\" ]]; then\n clone_result=$(git clone https://$git_user:${git_secret_token}@${git_url}/${git_project}/${git_repo}.git + $tempdir/$git_repo)\n elif [[ $repo_type == \"stash\" ]]; then\n #statements\n + \ if [[ $git_url_port != \"\" ]]; then\n clone_result=$(git clone https://$git_user:${git_secret_token}@${git_url}:$git_url_port/scm/${git_project}/$git_repo.git + $tempdir/$git_repo)\n else\n clone_result=$(git clone https://$git_user:${git_secret_token}@${git_url}/scm/${git_project}/$git_repo.git + $tempdir/$git_repo)\n fi\n fi\n echo $clone_result\n}\ngit_clone_ssh(){\n + \ echo \"cloning $git_project/$git_repo over ssh\"\n if [[ $repo_type == \"git\" + || $repo_type == \"bitbucket\" ]]\n then\n clone_result=$(git clone git@${git_url}:${git_project}/$git_repo.git + $tempdir/$git_repo)\n elif [[ $repo_type == \"stash\" && $git_url_port != \"\" + ]]; then\n #statements\n clone_result=$(git clone ssh://git@${git_url}:$git_url_port/${git_project}/$git_repo.git + $tempdir/$git_repo )\n else\n\n clone_result=$(git clone ssh://git@${git_url}:${git_project}/$git_repo.git + $tempdir/$git_repo $tempdir/$git_repo)\n fi\n echo $clone_result\n}\n\ngit_add_file() + {\n local file=$1\n git add $file\n}\n\ngit_add_all() {\n git add $1\n}\ngit_tag_all() + {\n git tag -a Backup-$TAGSTAMP -m \"$msg\"\n git push --tags\n}\n\ngit_delete_file() + {\n local file=$1\n git rm $file\n}\n\ngit_checkout_branch(){\n all_branches=$(git + branch -r| grep -w origin/$git_branch)\n echo $all_lbranches\n if [[ $all_branches + != \"\" ]]\n then\n branch_checkout_result=$(git checkout $git_branch)\n echo + $branch_checkout_result\n pull_requred=true\n else\n git checkout -b $git_branch\n + \ fi\n\n}\ngit_add_all(){\n\n\tgit add $1\n\n}\ngit_commit_all() {\n local branch=$git_branch\n + \ local msg=\"checking application and pipeline raw data\"\n if [ \"$pull_requred\" + = true ]; then\n git pull origin $branch --no-edit\n if [ \"$?\" != \"0\" + ];then\n echo \"[ERROR]: Failed to pull $branch upstream.\"\n exit 1\n + \ fi\nfi\n opts=\"\"\n if [ \"$git_commit_sign\" == \"true\" ]; then\n opts=\"-s\"\n + \ fi\n #git commit $opts -a -m $msg\n git commit -m \"$msg\"\n git push --set-upstream + origin $branch\n if [ \"$?\" != \"0\" ];then\n echo \"[ERROR]: Failed to push + $branch upstream.\"\n exit 1\n fi\n}\n\nsync_spin_to_git() {\n\n echo \"In + upload function which copies spinnaker application and pipeline from spinnaker + to repo\"\n\n local user_root_folder=$root_folder\n\n if [ \"$git_secret_sshkey\" + != \"\" ]; then\n git_clone_ssh\n elif [ \"$git_secret_token\" != \"\" ]; + then\n git_clone_http\n else\n echo \"git cloning requires either a git_aws_secret_key + to be set or git_aws_secret_token\"\n exit 5\n fi\n\n projectdir=$tempdir/$git_repo\n + \ cd $projectdir\n #We are done, get update git\n git_checkout_branch\n get_pipelines_data\n + \ git_add_all $root_folder\n git_commit_all\n return 0\n}\nsync_git_to_spin(){\n + \ setup_git\n if [ \"$git_secret_sshkey\" != \"\" ]; then\n git_clone_ssh\n + \ elif [ \"$git_secret_token\" != \"\" ]; then\n git_clone_http\n else\n echo + \"git cloning requires either a git_aws_secret_key to be set or git_aws_secret_token\"\n + \ exit 5\n fi\n projectdir=$tempdir/$git_repo\n cd $projectdir\n git_checkout_branch\n + \ syncup_spin\n}\n" + github.sh: | #!/bin/bash - gpg_dir=${HOME}/gpg - error=0 - api="https://api.github.com" - merge_sleep=10 - git_delete_branch_after_pr=true - # yaml_operation="destructive" # destroy, empty - ignore_errors=${ignore_spin_errors:-false} - - set_debug() { - local debug=$1 - if [[ $debug == "true" ]]; then - set -x - fi - } - - unset_debug() { - local debug=$1 - if [[ $debug == "true" ]]; then - set +x - fi - } - - setup_secret() { - local region=$1 - local secret=$2 - local target_dir=$3 - local target_file=$4 - # local sshdir=${HOME}/.ssh - - mkdir -p $target_dir - unset_debug $debug - sec=$(aws secretsmanager get-secret-value \ - --region $region \ - --secret-id $secret \ - --version-stage AWSCURRENT | \ - jq -r '.SecretString | fromjson | .[]') - echo $sec | openssl base64 -d -A -out $target_dir/$target_file - set_debug $debug - chmod 400 $target_dir/$target_file - # ssh -Tv git@github.com - } - - # Setup gpg for git signing - setup_gpg_for_git() { - echo "Setting up GPG for commit signing." - setup_secret $region $git_aws_secret_gpg_pub $gpg_dir pub.gpg - setup_secret $region $git_aws_secret_gpg_priv $gpg_dir priv.gpg - setup_secret $region $git_aws_secret_gpg_pass $gpg_dir phrase - mkdir -m 0700 -p $HOME/.gnupg - echo "pinentry-mode loopback" > $HOME/.gnupg/gpg.conf - echo "passphrase-file $gpg_dir/phrase" >> $HOME/.gnupg/gpg.conf - gpg --import $gpg_dir/priv.gpg; rm $gpg_dir/priv.gpg - gpg --import $gpg_dir/pub.gpg; rm $gpg_dir/pub.gpg - key=$(gpg --list-key | grep "^ " | awk '{ print $1 }' | tail -1) - git config --global user.signingkey $key - git config --global commit.gpgsign true - # git config --global push.gpgsign true - } - - # setup git for committing - setup_git() { - echo "Setting up the Git " - local name=$1 - local email=$2 - git config --global user.email "$email" - git config --global user.name "$name" - } - - git_clone_ssh() { - local dir=$1 - local repo=$2 - local project=$3 - - mkdir -p $dir - cd $dir/../ - git clone git@github.com:$repo/$project.git - cd - - } - - git_clone_ssh_change() { - local dir=$1 - local repo=$2 - local project=$3 - - mkdir -p $dir - cd $dir - git clone git@github.com:$repo/$project.git - cd - - } - - git_clone_http() { - local dir=$1 - local repo=$2 - local project=$3 - - mkdir -p $dir - cd $dir - rm -rf $3 - unset_debug $debug - echo "git from github.com $repo/$project over https" - git clone https://$git_friendly_username:$git_secret_token@github.com/$repo/$project - set_debug $debug - #cd - - } - - git_checkout() { - local dir=$1 - local git_checkout=$2 - - cd $dir - if [ "$git_checkout" != "" ]; then - git checkout $git_checkout + #this script funtions only work for github central repository + + #this script funtions only work for git repo + #env variables needed for this to work are as below + #***git_url="example.bitbucket.com" make sure you dont add http/https or / in the url + #****git_repo="pipelinepromotion" repo to be pushed/download pipeline json files from + #***git_project="kes" project key is needed to clone/push/pull merge code + #***git_user="tes.user" user is needed for cloning and pusing changes (stash does not support only access key) + #***git_branch="testbranch" the branch to which the code should be merged with + #***merge_branch=false if true then provide all the below env variables + # git_secret_token="dafjaljoahfoasjoijso" needed to create pull requests should be the git_users secret token + # git_pr_token="slkdfjaljoajfopaj" this is approver token to approve pull requests / you can also provide approver password here. + # git_approve_user="test.approver" username of the pull request approver + # git_password="adjoowddaw" make sure your password does not include special characters like # @*/. special characters cause git clone command to fail with https + # + # repo_type="stash" for selfhosted bitbucket server please use stash as repo type + #***root_folder="path/to/pipeline-promotion/folder" folder to be selected in the repo to which the pipeline jobs to be pushed + #***command=upload for running specific job - + # upload - to upload spinnaker pipeline json files to repo + # download - to download pipeline json file from repo and apply on spinnaker + #***spinnaker_applications="testapp,sampleapp" application needed to collect the pipeline information + #spinnaker_pipelines="" provide pipelines to be collected, if nothing given, all the pipelines of the application are collected + #git_secret_sshkey="sshkey" ssh key if you want to clone repo with ssh protocol + + # note *** env variables are mandatory to work with the script + + source scripts/git.sh + git_hub_api_url=$git_api_url + approve_pr_github(){ + approve_req=$(curl -o -I -L -s -w "%{http_code}" -X POST -H "Accept: application/vnd.github.v3+json" -u $git_approve_user:$git_pr_token $git_hub_api_url/$git_user/${git_repo}/pulls/${pr_id}/reviews \ + -d '{"body": "Spinnaker says LGTM","event": "APPROVE"}') + echo $approve_req + if [[ $approve_req == "200" ]];then + echo "merge request approved successfully" else - git branch -v - fi - cd - - } - - gen_branch_name() { - branch=$(cat /usr/share/dict/esperanto | grep '^[a-z]\+$' | shuf -n 1) - # branch=$(shuf -n1 /usr/share/dict/esperanto) - echo $branch - } - - git_branch() { - local branch=$1 - git branch $branch - git checkout $branch - } - - git_add_file() { - local file=$1 - git add $file - } - - git_add_all() { - git add . - } - - git_delete_file() { - local file=$1 - git rm $file - } - - git_commit() { - local branch=$1 - local file=$2 - shift; shift; - local msg=$@ - - opts="" - if [ "$git_commit_sign" == "true" ]; then - opts="-s" - fi - git commit $opts $file -m "$msg" - git push --set-upstream origin $branch - if [ "$?" != "0" ];then - echo "[ERROR]: Failed to push $branch upstream." + echo "FAIL: failed to approve the request $" exit 1 fi } - git_commit_all() { - local branch=$1 - local msg="checking application and pipeline raw data" + merge_pr_github(){ - opts="" - if [ "$git_commit_sign" == "true" ]; then - opts="-s" - fi - #git commit $opts -a -m $msg - git commit -m "$msg" - git push --set-upstream origin $branch - if [ "$?" != "0" ];then - echo "[ERROR]: Failed to push $branch upstream." + merge_req=$(curl -o -I -L -s -w "%{http_code}" -X PUT -H "Accept: application/vnd.github.v3+json" -u $git_user:$git_secret_token $git_hub_api_url/$git_user/${git_repo}/pulls/${pr_id}/merge) + echo $merge_req + if [[ $merge_req == "200" ]]; then + echo "merged pr successfully" + else + echo "FAILED: failed to merge $merge_pr" exit 1 - fi + fi } - # Here we use HUB... - git_pr_merge() { - local repo=$1 - local project=$2 - local branch=$3 - local main=${4:-master} - local label=${5:-spinnaker} - local msg=${6:-"Auto PR for $repo/$project $branch to $main"} + create_pr_github(){ - pr=$(hub pull-request \ - -b $repo:$main \ - -h $repo:$branch \ - -l spinnaker-pr \ - -m "$msg" | tail -1) - if [ "$?" != "0" -o "$pr" == "" ]; then - echo "PR Failed for $repo/$project $branch to $main" + local output=$(curl -X POST -H "Accept: application/vnd.github.v3+json" -u $git_user:$git_secret_token $git_hub_api_url/${git_user}/${git_repo}/pulls \ + -d '{"title": "pull request to merge '$git_branch' to master","body": "pull request to merge latest pipleine jobs information to '$target_branch'", "head": "'${git_branch}'","base": "'$target_branch'"}') + if [ "$?" != 0 ] + then + echo "master branch is already up-to-date" + exit 0 + else + echo $output + echo $output > pr_response.json + errors=$(cat pr_response.json| jq '(.errors)' | sed 's/\"//g') + if [[ $errors != null ]]; then + echo "ERROR: failed to raise pull request $errors" exit 1 - fi - # github only - prn=$(echo ${pr##*/}) - if [ "$prn" == "" ]; then - echo "Unable to figure out Pull Request Number" - exit 2 - fi - unset_debug $debug - review_url="$api/repos/$repo/$project/pulls/$prn/reviews" - res=$(curl \ - -XPOST \ - -H "Authorization: token $GITHUB_PR_TOKEN" \ - -d '{ - "body": "Spinnaker says LGTM", - "event": "APPROVE" - }' \ - $review_url) - if [ "$?" != "0" ]; then - echo "Review of $prn failed for $repo/$project $branch" - exit 3 - fi - sleep $merge_sleep - merge_url="$api/repos/$repo/$project/pulls/$prn/merge" - res=$(curl \ - -XPUT \ - -H "Authorization: token $GITHUB_PR_TOKEN" \ - $merge_url) - if [ "$?" != "0" ]; then - echo "Merge failed for $prn on $repo/$project $branch to $main" - exit 4 - fi - if [ "$git_delete_branch_after_pr" == "true" ]; then - db="$api/repos/$repo/$project/git/refs/heads/$branch" - curl \ - -XDELETE \ - -H "Authorization: token $GITHUB_PR_TOKEN" \ - $db - if [ "$?" != "0" ]; then - echo "Delete failed for $repo/$project $branch to $main" - exit 5 fi + pr_id=$(cat pr_response.json| jq '(.number)' | sed 's/\"//g') + if [[ $pr_id != "" ]]; then + echo "successfully created pull request " + else + echo "ERROR: failed to raise pull request $output" + exit 1 fi - set_debug $debug + fi } - mkdir_from_file() { - local file=$1 - - local dir=${file%/*} - if [ ! -d "$dir" ]; then - mkdir -p $dir + sync_spin_to_github(){ + setup_git + sync_spin_to_git + if [[ $merge_branch == "true" && $target_branch != "" && ($git_branch != $target_branch) ]];then + if [[ $git_api_url_port != "" ]];then + git_hub_api_url=$git_hub_api_url:$git_api_url_port + + create_pr_github + if [[ $auto_merge == "true" ]]; then + approve_pr_github + merge_pr_github + fi + else + create_pr_github + if [[ $auto_merge == "true" ]]; then + approve_pr_github + merge_pr_github + fi + fi fi } - - - - - - # upload function does copy spinnaker application and pipeline from github to s3 - upload_spin() { - - echo "In upload function which copies spinnaker application and pipeline from spinnaker to github" - - local user_root_folder=$HOME - - if [ "$git_secret_sshkey" != "" ]; then - git_clone_ssh_change $user_root_folder $git_repo $git_project - elif [ "$git_secret_token" != "" ]; then - git_clone_http $user_root_folder $git_repo $git_project - else - echo "git cloning requires either a git_aws_secret_key to be set or git_aws_secret_token" - exit 5 - fi - - projectdir=$HOME/$git_project - cd $projectdir - - spinnaker_app=$spinnaker_applications - IFS=',' read -r -a spinnaker_app_array <<< "$spinnaker_app" - - spinnaker_pipe=$spinnaker_pipelines - #IFS=',' read -r -a spinnaker_pipe_array <<< "k8s-deploy" - IFS=',' read -r -a spinnaker_pipe_array <<< "$spinnaker_pipe" - - for (( m=0; m<${#spinnaker_app_array[@]}; m++ )); do - sourceApp=${spinnaker_app_array[$m]} - echo -e "Processing application $sourceApp\n" - mkdir -p $sourceApp ; cd $sourceApp # Get into the correct directory - spin pipeline list --application $sourceApp > tmp.json - if [ "$?" != "0" ]; then - echo "ERROR: spin pipeline list --application $sourceApp" - return 1 - fi - cat tmp.json | jq '.[] | (.name)' | sed 's/\"//g' > pipelines_in_application.list - rm tmp.json - - spin application get $sourceApp > $sourceApp.json - if [ "$?" != "0" ]; then - echo "ERROR: spin application get $sourceApp" - return 1 - fi - - if [[ ${#spinnaker_pipe_array[@]} > 0 ]]; then - for (( p=0; p<${#spinnaker_pipe_array[@]}; p++ )); do - pipeLine=${spinnaker_pipe_array[$p]} - echo -e " Processing pipeline $pipeLine\n" - # Check if pipeline exists - existingPipe=`grep \^${pipeLine}\$ pipelines_in_application.list` - if [[ "$existingPipe" == "${pipeLine}" ]]; then - spin pipeline get --application $sourceApp --name "$pipeLine" > "$pipeLine.json" - if [ "$?" != "0" ]; then - echo "ERROR: spin spin pipeline get --application $sourceApp --name \"$pipeLine\"" - return 1 - fi - else - echo "WARNING: pipeline=${pipeLine} not found in application=$sourceApp ... skipping" - fi - done - else # No pipelines defined, get all the pipelines - while read -r line; do - echo -e " Processing pipeline $line\n" - spin pipeline get --application $sourceApp --name "$line" > "$line.json" - if [ "$?" != "0" ]; then - echo "ERROR: spin spin pipeline get --application $sourceApp --name $line" - return 1 - fi - done < pipelines_in_application.list - fi - if [[ "$pipelinecreateconf" == "true" ]]; then - create_default_params - fi - cd .. - done - - #We are done, get update git - git_add_all - git_commit_all - return 0 + s3.sh: | + #!/bin/bash + source scripts/spin.sh + absolute_path="$(dirname $(readlink -f $0))" + + # s3_folder=folder/in/s3/bucket if not given script uploads to root folder or the s3 bucket + # ***bucket_name=testenvpipelinebucket "bucktet name to upload pipeline configuration" + # ***AWS_ACCESS_KEY_ID="SKJGIHOBGIHIHOOH" access key to access s3 bucket + # ***AWS_SECRET_ACCESS_KEY="sdfjlasj2e334234sdljflsjflsd98y9sy/0UVv6eCg" secret to access s3 bucket + # ***repo_type=s3 provide repo type as s3 + #***command=upload for running specific job - + # upload - to upload spinnaker pipeline json files to repo + # download - to download pipeline json file from repo and apply on spinnaker + #***spinnaker_applications="testapp,sampleapp" application needed to collect the pipeline information + #spinnaker_pipelines="" provide pipelines to be collected, if nothing given, all the pipelines of the application are collected + + + # note *** env variables are mandatory to work with the script + s3_folder=$root_folder + tempdir="/tmp/" + bucket_name=$repo_name + create_bucket(){ + #to create a bucket in s3 bucket name needed + aws s3 mb s3://$bucket_name + if [ $? != 0 ]; then + echo "[ERROR]: Failed to create s3 bucket might be aleady existing" + fi } - download_spin() { - echo "In Download function that updates the spinnaker instance with the contents in git" - local user_root_folder=$HOME + list_bucket(){ + # to llst bucket objects + aws s3 ls s3://$bucket_name/ + if [ $? != 0 ]; then + echo "[ERROR]: Failed to list s3 bucket " + fi + } + list_application_folder(){ + # to list an object folder + aws ls s3://$bucket_name/${s_folder}/$1 | awk '{print $4}' + } - if [ "$git_secret_sshkey" != "" ]; then - git_clone_ssh_change $user_root_folder $git_repo $git_project - elif [ "$git_secret_token" != "" ]; then - git_clone_http $user_root_folder $git_repo $git_project + upload_spin_to_s3(){ + # get the pipeline data from spinnaker and store in root_folder + get_pipelines_data + #upload spinnaker pipelines data and upload to s3 folder + aws s3 cp $tempdir$s3_folder s3://$bucket_name/$s3_folder --recursive + if [ "$?" != 0 ]; then + echo "[ERROR]: Failed to upload to bucket" $bucket_name else - echo "git cloning requires either a git_secret_sshkey to be set or git_secret_token" - exit 5 + echo "uploaded to bucket successfully" fi - - projectdir=$HOME/$git_project - cd $projectdir - - spinnaker_app=$spinnaker_applications - IFS=',' read -r -a spinnaker_app_array <<< "$spinnaker_app" - - spinnaker_pipe=$spinnaker_pipelines - #IFS=',' read -r -a spinnaker_pipe_array <<< "k8s-deploy" - IFS=',' read -r -a spinnaker_pipe_array <<< "$spinnaker_pipe" - - - for (( m=0; m<${#spinnaker_app_array[@]}; m++ )); do - sourceApp=${spinnaker_app_array[$m]} - echo -e "Processing application $sourceApp\n" - cd $sourceApp # Get into the correct directory - if [ "$?" != "0" ]; then - echo "ERROR: Unable to change to application directory: $sourceApp" - return 1 - fi - - #Create the application by default, we can have flag to for this later - spin application save -f $sourceApp.json - retVal=$? - if [[ "$retVal" != "0" && "$ignore_errors" == "false" ]]; then - echo "ERROR: spin application save $sourceApp" - return 1 - elif [[ "$retVal" != "0" && "$ignore_errors" == "true" ]]; then - echo "ERROR: spin application save $sourceApp, continuing" - cd .. - continue - fi - sleep 30 # Give a few seconds after application creation - - if [[ ${#spinnaker_pipe_array[@]} > 0 ]]; then - for (( p=0; p<${#spinnaker_pipe_array[@]}; p++ )); do - pipeLine=${spinnaker_pipe_array[$p]} - echo -e " Processing pipeline $pipeLine\n" - # Check if pipeline file exists - if [ -f "$pipeLine.json" ]; then - #Update parameterConfig - if [[ "$pipelineconfig" == "true" ]]; then - mkdir -p temp - update_params "$pipeLine.json" - rm -rf temp - fi - spin pipeline save --file "$pipeLine.json" - retVal=$? - if [[ "$retVal" != "0" && "$ignore_errors" == "false" ]]; then - echo "ERROR: spin pipeline save --file $pipeLine.json" - return 1 - elif [[ "$retVal" != "0" && "$ignore_errors" == "true" ]]; then - echo "ERROR: spin pipeline save --file $pipeLine.json, continuing" - continue - fi - else - echo "WARNING: pipeline=${pipeLine} not found in application=$sourceApp ... skipping" - fi - done - else # No pipelines defined, get all the pipelines - while read -r line; do - [[ -f "$line.json" ]] || continue - pipeLine=$line - echo -e " Processing pipeline $pipeLine\n" - - #Update parameterConfig - if [[ "$pipelineconfig" == "true" ]]; then - mkdir -p temp - update_params "$pipeLine.json" - #rm -rf temp - fi - spin pipeline save --file "$pipeLine.json" - retVal=$? - if [[ "$retVal" != "0" && "$ignore_errors" == "false" ]]; then - echo "ERROR: spin pipeline save --file $pipeLine.json" - return 1 - elif [[ "$retVal" != "0" && "$ignore_errors" == "true" ]]; then - echo "ERROR: spin pipeline save --file $pipeLine.json, continuing" - continue - fi - sleep 5 # Slow it down - done < pipelines_in_application.list - fi - cd .. - done - } + sync_from_s3_spin(){ - update_params() { - confDir=${pipelineconfigdir} - if [ ! -d "$confDir" ] ; then - echo "Directory specified for configuratio ($confDir) not found in application directory" - return - fi - json="$1" - - echo "Processing pipeline ($json) and updating pipelines as per configuration in $confDir" - - #Extract .parameterConfig - cat "$json" | jq '.parameterConfig' > temp/"config-$json" - - #Replace parameters - cat temp/"config-$json" | jq -f /home/opsmx/scripts/replace-params.jq --argfile pp $confDir/"$json" > temp/"updated-config-$json" - - #Replace .parameterConfig - cat "$json" | jq '.parameterConfig=$uc' --argfile uc temp/"updated-config-$json" > temp/"$json" - - ######################################################################## - #Extract 1st trigger - cat temp/"$json"| jq '.triggers[0]' > temp/tmp-trig.json - - #Update first trigger - cat temp/tmp-trig.json | jq 'if $pp.triggerValues != null then . * $pp.triggerValues else . end' --argfile pp $confDir/"$json" > temp/updated-tmp-trig.json - - #Update pipeline-json with updated trigger - if [[ `cat temp/updated-tmp-trig.json | wc -c` -gt 5 ]] - then - cat temp/"$json" | jq '.triggers[0]=$pp' --argfile pp temp/updated-tmp-trig.json > temp/final-replaced.json - cp temp/final-replaced.json "$json" - else - cp temp/"$json" "$json" - fi - - ######################################################################## + echo "downloading spinnaker application pipelines configuration" + aws s3 sync s3://$bucket_name/$s3_folder $tempdir$s3_folder + #apply configuration in spinnaker + syncup_spin } - #Create default parameterconfig-files - create_default_params() { - targetDir=${1:-default-config} - echo "Processing pipelines and creating output in $targetDir" - mkdir -p $targetDir - for json in *.json ; do - [[ -f "$json" ]] || continue - echo " processing $json" - cat "$json" | jq '.parameterConfig | reduce .[] as $p ({};.Parameters += {($p.name): $p.default})' > $targetDir/tmp-param.json 2>/dev/null - cat "$json" | jq '.triggers[0] ' > $targetDir/tmp-trig.json 2>/dev/null - - if [[ `cat $targetDir/tmp-trig.json | wc -c` -gt 5 ]] - then - cat $targetDir/tmp-param.json | jq '.triggerValues=$pp' --argfile pp $targetDir/tmp-trig.json > $targetDir/"$json" 2>/dev/null - else - cp $targetDir/tmp-param.json $targetDir/"$json" - fi - done - rm -f $targetDir/tmp-param.json - rm -f $targetDir/tmp-trig.json - #Remove all files with zero size - echo "Removing files that do not have any parameters defined" - find $targetDir -type f -size -4c -delete # No parameterConfig in the file - #find $targetDir -type f -size -4c -print -delete # No parameterConfig in the file + delete_s3_object(){ + #delete an object in bucket + aws rm s3://$bucket_name/${s3_folder}/${application_name}/ --recursive + if [ $? != 0 ]; then + echo "[ERROR]: Failed to delete s3 application folder " + else + echo "created bucket successfully" + fi } - - - setup_git $git_friendly_username $git_user_email - if [[ "$command" == "download" ]]; then - download_spin - elif [[ "$command" == "upload" ]]; then - upload_spin - else - echo "command is not set or is invalid ($command)" - exit 5 - fi - exit $? - replace-params.jq: | - .[] # Stream input json (orig-paramConfig) - | .default= # set value of "default" key to either the new one or the old one - ( . as $paramFull # save the current input object as paramFull , this is the entire parameterConfig block - | $pp.Parameters # strea new-configs - | if has($paramFull.name) then # If the new-configs has a key that is the value of the "name" key of the existing config - $pp.Parameters[$paramFull.name] # return the new value - else - $paramFull.default # else simply return the current value - end - ) -{{- end }} + spin.sh: "#!/bin/bash\n#source $(dirname $0)/git.sh\ntempdir=\"/tmp/\"\n\n#spinnaker_applications=\"sampleapp\"\nget_app_pipelines(){\n\tspin + pipeline list --application $1 > tmp.json\n\tif [ \"$?\" != \"0\" ]; then\n\t\t\techo + \"ERROR: spin pipeline list --application $1\"\n\t\t\treturn 1\n\tfi\n\tcat tmp.json + | jq '.[] | (.name)' | sed 's/\\\"//g' > pipelines_in_application.list\n\tcat + tmp.json | jq '.[] | (.id)' | sed 's/\\\"//g' > pipelines_guid.list\n\trm tmp.json\n}\n\n\nlive_backup_spin() + {\n\n#This function will backup existing spinnaker data and store it in local + for further comparison\n\n if [[ $repo_type = \"s3\" ]]; then\n\t\tprojectdir=$tempdir/$root_folder\n\telse\n\t\tprojectdir=$tempdir/${git_repo}/$root_folder\n + \ fi\n\tlive_projectdir_workdir=$projectdir/live_backup\n\n if [ -d \"$live_projectdir_workdir\" + ]\n then\n echo \"given live_spinnaker_project_work_dir is present\"\n else\n + \ echo \"given live_spinnaker_project_work_dir is not present therefore creating + it\"\n mkdir -p \"$projectdir/live_backup\"\n fi\n\n cd $live_projectdir_workdir\n\n + \ spinnaker_app=$spinnaker_applications\n IFS=',' read -r -a spinnaker_app_array + <<< \"$spinnaker_app\"\n\n\n spinnaker_pipe=$spinnaker_pipelines\n IFS=',' read + -r -a spinnaker_pipe_array <<< \"$spinnaker_pipe\"\n\n for (( m=0; m<${#spinnaker_app_array[@]}; + m++ )); do\n sourceApp=${spinnaker_app_array[$m]}\n echo -e \"Processing + application $sourceApp\\n\"\n mkdir -p $sourceApp ; cd $sourceApp\n\t\t # + Get into the correct directory\n spin -k pipeline list --application $sourceApp + \ > tmp.json\n if [ \"$?\" != \"0\" ]; then\n echo \"ERROR: spin pipeline + list --application $sourceApp\"\n return 1\n fi\n cat tmp.json + | jq '.[] | (.name)' | sed 's/\\\"//g' > live_pipelines_in_application.list\n + \ cat tmp.json | jq '.[] | (.id)' | sed 's/\\\"//g' > live_pipelines_guid.list\n + \ rm tmp.json\n\n spin -k application get $sourceApp > $sourceApp.json\n + \ if [ \"$?\" != \"0\" ]; then\n echo \"ERROR: spin application get + $sourceApp\"\n return 1\n fi\n\n if [[ ${#spinnaker_pipe_array[@]} + > 0 ]]; then\n for (( p=0; p<${#spinnaker_pipe_array[@]}; p++ )); do\n + \ pipeLine=${spinnaker_pipe_array[$p]}\n echo -e \" Processing + pipeline $pipeLine\\n\"\n # Check if pipeline exists\n existingPipe=`grep + \\^${pipeLine}\\$ live_pipelines_in_application.list`\n if [[ \"$existingPipe\" + == \"${pipeLine}\" ]]; then\n spin -k pipeline get --application + $sourceApp --name \"$pipeLine\" > \"$pipeLine.json\"\n if [ \"$?\" + != \"0\" ]; then\n echo \"ERROR: spin spin pipeline get --application + $sourceApp --name \\\"$pipeLine\\\"\"\n return 1\n fi\n + \ else\n echo \"WARNING: pipeline=${pipeLine} not found + in application=$sourceApp ... skipping\"\n fi\n done\n else + # No pipelines defined, get all the pipelines\n while read -r line; do\n + \ echo -e \" Processing pipeline $line\\n\"\n spin -k + pipeline get --application $sourceApp --name \"$line\" > \"$line.json\"\n if + [ \"$?\" != \"0\" ]; then\n echo \"ERROR: spin spin pipeline get + --application $sourceApp --name $line\"\n return 1\n fi\n + \ done < live_pipelines_in_application.list\n fi\n cd ..\n done\n + \ return 0\n}\n\ndelete_odd_pipelines() {\n #Delete the additional pielines that + are in spinnaker and not in git\n for (( m=0; m<${#spinnaker_app_array[@]}; + m++ )); do\n\t sourceApp=${spinnaker_app_array[$m]}\n\t if [ -f \"$projectdir/live_backup/$sourceApp/odd_pipeline.txt\" + ]; then\n if [ ! -s \"$projectdir/live_backup/$sourceApp/odd_pipeline.txt\" + ]; then\n\t echo \"no new pipelines to delete\"\n else\n echo + \"============ Delete pipeline in $sourceApp Application =============\"\n\n\t + \ while IFS= read -r pipelinename; do\n echo \"Deleting the pipeline + $pipelinename\"\n\t spin -k pipeline delete --name $pipelinename --application + $sourceApp\n done < $projectdir/live_backup/$sourceApp/odd_pipeline.txt\n\n\t + \ rm -rf $projectdir/live_backup/$sourceApp/odd_pipeline.txt\n\n fi\n fi\ndone\n\n}\n#Create + default parameterconfig-files\ncreate_default_params() {\n targetDir=${1:-default-config}\n + \ echo \"Processing pipelines and creating output in $targetDir\"\n mkdir + -p $targetDir\n for json in *.json ; do\n [[ -f \"$json\" ]] || continue\n + \ echo \"\tprocessing $json\"\n cat \"$json\" | jq '.parameterConfig + | reduce .[] as $p ({};.Parameters += {($p.name): $p.default})' > $targetDir/tmp-param.json + 2>/dev/null\n cat \"$json\" | jq '.triggers[0] ' > $targetDir/tmp-trig.json + 2>/dev/null\n\n if [[ `cat $targetDir/tmp-trig.json | wc -c` -gt 5 ]]\n then\n + \ cat $targetDir/tmp-param.json | jq '.triggerValues=$pp' --argfile pp $targetDir/tmp-trig.json + > $targetDir/\"$json\" 2>/dev/null\n else\n cp $targetDir/tmp-param.json + $targetDir/\"$json\"\n fi\n done\n rm -f $targetDir/tmp-param.json\n + \ rm -f $targetDir/tmp-trig.json\n #Remove all files with zero size\n echo + \"Removing files that do not have any parameters defined\"\n find $targetDir + -type f -size -4c -delete # No parameterConfig in the file\n #find $targetDir + -type f -size -4c -print -delete # No parameterConfig in the file\n}\n\nequate_pipelines_in_app() + {\n\n #This function will comapre the applications and pipelines in git and spinnaker + and gives the additional pipelines data\n\n IFS=',' read -r -a spinnaker_app_array + <<< \"$spinnaker_app\"\n\n IFS=',' read -r -a spinnaker_pipe_array <<< \"$spinnaker_pipe\"\n\n + \ for (( m=0; m<${#spinnaker_app_array[@]}; m++ )); do\n sourceApp=${spinnaker_app_array[$m]}\n\n + \ touch $projectdir/live_backup/$sourceApp/odd_pipeline_id.txt\n\n\t\t echo + $projectdir\n\t\t echo $git_project_work_dir\n\t\t echo $sourceApp\n diff + $projectdir/$git_project_work_dir/$sourceApp/pipelines_guid.list $projectdir/live_backup/$sourceApp/live_pipelines_guid.list + | awk '{print $2}' | sed 1d > $projectdir/live_backup/$sourceApp/odd_pipeline_id.txt\n\n + \ #list all existing spinnaker pipelines with app as reference\n spin -k + pipeline list --application $sourceApp > $projectdir/live_backup/$sourceApp/$sourceApp-pipeline_list.json\n + \ touch $projectdir/live_backup/$sourceApp/odd_pipeline.txt\n\n while IFS= + read -r id; do\n #Extract the pipeline names using guids as reference\n cat + $projectdir/live_backup/$sourceApp/$sourceApp-pipeline_list.json | jq '.[] | select + (.id==\"'$id'\") | .name' -r >> $projectdir/live_backup/$sourceApp/odd_pipeline.txt\n + \ done < $projectdir/live_backup/$sourceApp/odd_pipeline_id.txt\n done\n}\n\nsyncup_spin() + {\n echo \"In Download function that updates the spinnaker instance with the + contents in repo\"\n\n #Backup of existing spinnaker pipelines with guids\n live_backup_spin\n\n + \ #Compare guids of existing pipelines and pipelines in git and provide names + of additional pipelines\n equate_pipelines_in_app\n\n #Delete the extra pipelines(pipelines + in spinnaker and not in git)\n\tif [[ $delete_on_sync_spin == \"true\" ]]; then\n\t\tdelete_odd_pipelines\n\tfi\n\n\tif + [[ $repo_type = \"s3\" ]]; then\n\t\tprojectdir=$tempdir/$root_folder\n\t\techo + \"project dir at synup spin $projectdir\"\n\telse\n\n\t\tprojectdir=$tempdir/${git_repo}/$root_folder\n\tfi\n + \ if [ -d \"$projectdir\" ]\n then\n echo \"given git_project_work_dir is + present\"\n else\n echo \"given git_project_work_dir is not present therefore + creating it\"\n mkdir -p \"$projectdir/$git_project_work_dir\"\n fi\n\n cd + $projectdir\n spinnaker_app=$spinnaker_applications\n IFS=',' read -r -a spinnaker_app_array + <<< \"$spinnaker_app\"\n\n spinnaker_pipe=$spinnaker_pipelines\n #IFS=',' read + -r -a spinnaker_pipe_array <<< \"k8s-deploy\"\n IFS=',' read -r -a spinnaker_pipe_array + <<< \"$spinnaker_pipe\"\n\n echo $projectdir\n for (( m=0; m<${#spinnaker_app_array[@]}; + m++ )); do\n sourceApp=${spinnaker_app_array[$m]}\n echo -e \"Processing + application $sourceApp\\n\"\n cd $sourceApp # Get into the correct + directory\n if [ \"$?\" != \"0\" ]; then\n echo \"ERROR: Unable to + change to application directory: $sourceApp\"\n return 1\n fi\n\n + \ #Create the application by default, we can have flag to for this later\n + \ spin -k application save -f $sourceApp.json\n retVal=$?\n if [[ \"$retVal\" + != \"0\" && \"$ignore_errors\" == \"false\" ]]; then\n echo \"ERROR: spin + application save $sourceApp\"\n return 1\n elif [[ \"$retVal\" != + \"0\" && \"$ignore_errors\" == \"true\" ]]; then\n echo \"ERROR: spin + application save $sourceApp, continuing\"\n cd ..\n continue\n + \ fi\n #sleep 30 # Give a few seconds after application creation\n\n if + [[ ${#spinnaker_pipe_array[@]} > 0 ]]; then\n for (( p=0; p<${#spinnaker_pipe_array[@]}; + p++ )); do\n pipeLine=${spinnaker_pipe_array[$p]}\n echo + -e \" Processing pipeline $pipeLine\\n\"\n # Check if pipeline file + \ exists\n if [ -f \"$pipeLine.json\" ]; then\n #Update + parameterConfig\n if [[ \"$pipelineconfig\" == \"true\" ]]; then\n\n + \ mkdir -p temp\n update_params \"$pipeLine.json\"\n + \ rm -rf temp\n fi\n spin -k pipeline + save --file \"$pipeLine.json\"\n retVal=$?\n if [[ + \"$retVal\" != \"0\" && \"$ignore_errors\" == \"false\" ]]; then\n echo + \"ERROR: spin pipeline save --file $pipeLine.json\"\n return + 1\n elif [[ \"$retVal\" != \"0\" && \"$ignore_errors\" == \"true\" + ]]; then\n echo \"ERROR: spin pipeline save --file $pipeLine.json, + continuing\"\n continue\n fi\n else\n + \ echo \"WARNING: pipeline=${pipeLine} not found in application=$sourceApp + ... skipping\"\n fi\n done\n else # No pipelines defined, + get all the pipelines\n while read -r line; do\n [[ -f \"$line.json\" + ]] || continue\n pipeLine=$line\n echo -e \" Processing + pipeline $pipeLine\\n\"\n\n #Update parameterConfig\n if + [[ \"$pipelineconfig\" == \"true\" ]]; then\n\t\t\t\t\t\t\techo \"in pipelineconfig + else\"\n mkdir -p temp\n update_params \"$pipeLine.json\"\n + \ #rm -rf temp\n fi\n\n echo `realpath $pipeLine.json`\n\t\t\t\t\t\tif + test -f \"$pipeLine.json\"; then\n\t\t\t\t\t\t\tspin -k pipeline save --file \"$pipeLine.json\"\n\t\t\t\t\t\tfi\n\n + \ retVal=$?\n if [[ \"$retVal\" != \"0\" && \"$ignore_errors\" + == \"false\" ]]; then\n echo \"ERROR: spin pipeline save --file + $pipeLine.json\"\n return 1\n elif [[ \"$retVal\" != + \"0\" && \"$ignore_errors\" == \"true\" ]]; then\n echo \"ERROR: + spin pipeline save --file $pipeLine.json, continuing\"\n continue\n + \ fi\n sleep 5 # Slow it down\n done < pipelines_in_application.list\n + \ fi\n cd ..\n done\n\n}\nget_pipelines_data(){\n\t spinnaker_app=$spinnaker_applications\n + \ IFS=',' read -r -a spinnaker_app_array <<< \"$spinnaker_app\"\n spinnaker_pipe=$spinnaker_pipelines\n + \ #IFS=',' read -r -a spinnaker_pipe_array <<< \"k8s-deploy\"\n IFS=',' + read -r -a spinnaker_pipe_array <<< \"$spinnaker_pipe\"\n\n\t\t\t\tif [[ $root_folder + == \"\" ]]; then\n\t\t\t\t\troot_folder=\".\"\n\t\t\t\tfi\n for (( m=0; + m<${#spinnaker_app_array[@]}; m++ )); do\n sourceApp=${spinnaker_app_array[$m]}\n + \ echo -e \"Processing application $sourceApp\\n\"\n\n\t\t echo \"get pipelines + data $root_folder\"\n mkdir -p $tempdir/$git_repo/${root_folder}/$sourceApp + ; cd $tempdir/$git_repo/${root_folder}/$sourceApp # Get into the + correct directory\n\n get_app_pipelines $sourceApp\n spin application + get $sourceApp > $sourceApp.json\n if [ \"$?\" != \"0\" ]; then\n echo + \"ERROR: spin application get $sourceApp\"\n return 1\n fi\n if + [[ ${#spinnaker_pipe_array[@]} > 0 ]]; then\n for (( p=0; p<${#spinnaker_pipe_array[@]}; + p++ )); do\n pipeLine=${spinnaker_pipe_array[$p]}\n echo + -e \" Processing pipeline $pipeLine\\n\"\n # Check if pipeline exists\n + \ existingPipe=`grep \\^${pipeLine}\\$ pipelines_in_application.list`\n + \ if [[ \"$existingPipe\" == \"${pipeLine}\" ]]; then\n spin + pipeline get --application $sourceApp --name \"$pipeLine\" > \"$pipeLine.json\"\n\n + \ if [ \"$?\" != \"0\" ]; then\n echo \"ERROR: + spin spin pipeline get --application $sourceApp --name \\\"$pipeLine\\\"\"\n + \ return 1\n fi\n else\n echo + \"WARNING: pipeline=${pipeLine} not found in application=$sourceApp ... skipping\"\n + \ fi\n done\n else # No pipelines defined, get all the pipelines\n + \ while read -r line; do\n echo -e \" Processing pipeline + $line\\n\"\n spin pipeline get --application $sourceApp --name \"$line\" + > \"$line.json\"\n if [ \"$?\" != \"0\" ]; then\n echo + \"ERROR: spin spin pipeline get --application $sourceApp --name $line\"\n return + 1\n fi\n\n done < pipelines_in_application.list\n fi\n + \ if [[ \"$pipelinecreateconf\" == \"true\" ]]; then\n create_default_params\n + \ fi\n cd -\n done\n}\n\ndownload_spin() {\n echo \"In Download function + that updates the spinnaker instance with the contents in git\"\n local user_root_folder=$root_folder\n\n + \ if [ \"$git_secret_sshkey\" != \"\" ]; then\n git_clone_ssh_change $user_root_folder + $git_repo $git_project\n elif [ \"$git_secret_token\" != \"\" ]; then\n git_clone_http + $user_root_folder $git_repo $git_project\n else\n echo \"git cloning requires + either a git_secret_sshkey to be set or git_secret_token\"\n exit 5\n fi\n\n + \ projectdir=$HOME/$git_project\n cd $projectdir\n\n spinnaker_app=$spinnaker_applications\n + \ IFS=',' read -r -a spinnaker_app_array <<< \"$spinnaker_app\"\n\n spinnaker_pipe=$spinnaker_pipelines\n + \ #IFS=',' read -r -a spinnaker_pipe_array <<< \"k8s-deploy\"\n IFS=',' read + -r -a spinnaker_pipe_array <<< \"$spinnaker_pipe\"\n\n\n for (( m=0; m<${#spinnaker_app_array[@]}; + m++ )); do\n sourceApp=${spinnaker_app_array[$m]}\n echo -e \"Processing + application $sourceApp\\n\"\n cd $sourceApp # Get into the correct + directory\n if [ \"$?\" != \"0\" ]; then\n echo \"ERROR: Unable to + change to application directory: $sourceApp\"\n return 1\n fi\n\n + \ #Create the application by default, we can have flag to for this later\n + \ spin application save -f $sourceApp.json\n retVal=$?\n if [[ \"$retVal\" + != \"0\" && \"$ignore_errors\" == \"false\" ]]; then\n echo \"ERROR: spin + application save $sourceApp\"\n return 1\n elif [[ \"$retVal\" != + \"0\" && \"$ignore_errors\" == \"true\" ]]; then\n echo \"ERROR: spin + application save $sourceApp, continuing\"\n cd ..\n continue\n + \ fi\n sleep 30 # Give a few seconds after application creation\n\n if + [[ ${#spinnaker_pipe_array[@]} > 0 ]]; then\n for (( p=0; p<${#spinnaker_pipe_array[@]}; + p++ )); do\n pipeLine=${spinnaker_pipe_array[$p]}\n echo + -e \" Processing pipeline $pipeLine\\n\"\n # Check if pipeline file + \ exists\n if [ -f \"$pipeLine.json\" ]; then\n #Update + parameterConfig\n if [[ \"$pipelineconfig\" == \"true\" ]]; then\n + \ mkdir -p temp\n update_params \"$pipeLine.json\"\n + \ rm -rf temp\n fi\n spin pipeline + save --file \"$pipeLine.json\"\n retVal=$?\n if [[ + \"$retVal\" != \"0\" && \"$ignore_errors\" == \"false\" ]]; then\n echo + \"ERROR: spin pipeline save --file $pipeLine.json\"\n return + 1\n elif [[ \"$retVal\" != \"0\" && \"$ignore_errors\" == \"true\" + ]]; then\n echo \"ERROR: spin pipeline save --file $pipeLine.json, + continuing\"\n continue\n fi\n else\n + \ echo \"WARNING: pipeline=${pipeLine} not found in application=$sourceApp + ... skipping\"\n fi\n done\n else # No pipelines defined, + get all the pipelines\n while read -r line; do\n [[ -f \"$line.json\" + ]] || continue\n pipeLine=$line\n echo -e \" Processing + pipeline $pipeLine\\n\"\n\n #Update parameterConfig\n if + [[ \"$pipelineconfig\" == \"true\" ]]; then\n mkdir -p temp\n update_params + \"$pipeLine.json\"\n #rm -rf temp\n fi\n spin + pipeline save --file \"$pipeLine.json\"\n retVal=$?\n if + [[ \"$retVal\" != \"0\" && \"$ignore_errors\" == \"false\" ]]; then\n echo + \"ERROR: spin pipeline save --file $pipeLine.json\"\n return 1\n + \ elif [[ \"$retVal\" != \"0\" && \"$ignore_errors\" == \"true\" ]]; + then\n echo \"ERROR: spin pipeline save --file $pipeLine.json, + continuing\"\n continue\n fi\n sleep 5 # + Slow it down\n done < pipelines_in_application.list\n fi\n cd + ..\n done\n\n}\n\nupdate_params() {\n confDir=${pipelineconfigdir}\n if + [ ! -d \"$confDir\" ] ; then\n echo \"Directory specified for configuratio + ($confDir) not found in application directory\"\n return\n fi\n if + [ ! -f \"$confDir/$json\" ] ; then\n echo \"INFO: No configuration found + for $json in $confDir\"\n return\n fi\n json=\"$1\"\n echo \"Processing + pipeline ($json) and updating pipelines as per configuration in $confDir\"\n #Extract + .parameterConfig\n cat \"$json\" | jq '.parameterConfig' > temp/\"config-$json\"\n + \ #Replace parameters\n cat temp/\"config-$json\" | jq -f /home/opsmx/scripts/replace-params.jq + --argfile pp $confDir/\"$json\" > temp/\"updated-config-$json\"\n\n #Replace + .parameterConfig\n cat \"$json\" | jq '.parameterConfig=$uc' --argfile uc + temp/\"updated-config-$json\" > temp/\"$json\"\n\n ########################################################################\n + \ #Extract 1st trigger\n cat temp/\"$json\"| jq '.triggers[0]' > temp/tmp-trig.json\n + \ #Update first trigger\n cat temp/tmp-trig.json | jq 'if $pp.triggerValues + != null then . * $pp.triggerValues else . end' --argfile pp $confDir/\"$json\" + \ > temp/updated-tmp-trig.json\n #Update pipeline-json with updated trigger\n + \ if [[ `cat temp/updated-tmp-trig.json | wc -c` -gt 5 ]]\n then\n cat + temp/\"$json\" | jq '.triggers[0]=$pp' --argfile pp temp/updated-tmp-trig.json + > temp/final-replaced.json\n cp temp/final-replaced.json \"$json\"\n else\n + \ cp temp/\"$json\" \"$json\"\n fi\n ########################################################################\n}\n" + stash.sh: "#!/bin/bash\n\n#this script funtions only work for self hosted bitbucketserver/stash + central repository\n#env variables needed for this to work are as below\n#***git_url=\"example.bitbucket.com\" + make sure you dont add http/https or / in the url\n#****git_repo=\"pipelinepromotion\" + repo to be pushed/download pipeline json files from\n#***git_project=\"kes\" project + key is needed to clone/push/pull merge code\n#***git_user=\"tes.user\" user is + needed for cloning and pusing changes (stash does not support only access key)\n#git_password=\"adjoowddaw\" + make sure your password does not include special characters like # @*/. special + characters cause git clone command to fail with https\n#***git_branch=\"testbranch\" + the branch to which the code should be merged with\n#***merge_branch=false if + true then provide all the below env variables\n# git_secret_token=\"dafjaljoahfoasjoijso\" + needed to create pull requests should be the git_users secret token\n# git_pr_token=\"slkdfjaljoajfopaj\" + this is approver token to approve pull requests / you can also provide approver + password here.\n# git_approve_user=\"test.approver\" username of the pull request + approver\n#\n# repo_type=\"stash\" for selfhosted bitbucket server please use + stash as repo type\n#***root_folder=\"path/to/pipeline-promotion/folder\" folder + to be selected in the repo to which the pipeline jobs to be pushed\n#***command=upload + for running specific job -\n# upload - + to upload spinnaker pipeline json files to repo\n# download + - to download pipeline json file from repo and apply on spinnaker\n#***spinnaker_applications=\"testapp,sampleapp\" + application needed to collect the pipeline information\n#spinnaker_pipelines=\"\" + provide pipelines to be collected, if nothing given, all the pipelines of the + application are collected\n#git_secret_sshkey=\"sshkey\" ssh key if you want to + clone repo with ssh protocol\n\n# note *** env variables are mandatory to work + with the script\n\nsource scripts/git.sh\ngit_repo=$repo_name\npr_id=0\npr_version=0\napprove_pr_stash(){\n + \ approve_req=$(curl -k -o -I -L -s -w \"%{http_code}\" -X POST -H \"Content-Type: + application/json\" -u $git_approve_user:$git_pr_token \\\n https://$git_api_url/${git_project}/repos/${git_repo}/pull-requests/${pr_id}/approve)\n + \ echo $approve_req\n if [[ $approve_req == \"200\" ]];then\n echo \"merge + request approved successfully\"\n else\n echo \"FAIL: failed to approve the + request \"\n exit 1\n fi\n}\n\nmerge_pr_stash(){\n\n merge_req=$(curl -k + -o -I -L -s -w \"%{http_code}\" -X POST -H \"Content-Type: application/json\" + -u $git_user:$git_secret_token \\\n https://$git_api_url/${git_project}/repos/${git_repo}/pull-requests/${pr_id}/merge?version=$pr_version)\n + \ echo $merge_req\n if [ $merge_req == \"200\" ]; then\n echo \"merged pr + successfully\"\n else\n echo \"FAILED: failed to merge $merge_pr\"\n exit + 1\nfi\n}\ncreate_pr_stash(){\n\n\tlocal output=$(curl -k -X POST -H \"Content-Type: + application/json\" -u $git_user:$git_secret_token https://$git_api_url/${git_project}/repos/${git_repo}/pull-requests + -d '{\n \"title\": \"merging '\"$git_branch\"' to '\"$target_branch\"'\",\n + \ \"description\": \"changes from spinnaker pipeline jobs are to be merged to + master\",\n \"state\": \"OPEN\",\n \"open\": true,\n \"closed\": false,\n + \ \"fromRef\": {\n \"id\": \"refs/heads/'\"${git_branch}\"'\",\n \"repository\": + {\n \"slug\": \"'\"${git_repo}\"'\",\n \"name\": null,\n + \ \"project\": {\n \"key\": \"'\"${git_project}\"'\"\n + \ }\n }\n },\n \"toRef\": {\n \"id\": \"refs/heads/'\"$target_branch\"'\",\n + \ \"repository\": {\n \"slug\": \"'\"${git_repo}\"'\",\n \"name\": + null,\n \"project\": {\n \"key\": \"'\"${git_project}\"'\"\n + \ }\n }\n },\n \"locked\": false\n}')\n echo $output\n + \ echo $output > pr_response.json\n grep \"is already up-to-date with branch\" + pr_response.json\n if [ \"$?\" = 0 ]\n then\n echo \"master branch is already + up-to-date\"\n exit 0\n else\n pr_id=$(cat pr_response.json| jq '(.id)' + | sed 's/\\\"//g')\n pr_version=$(cat pr_response.json | jq '(.version)' | + sed 's/\\\"//g')\n\n if [ $? = 0 ]; then\n echo \"successfully created + pull request \"\n #rm -f pr_response.json\n else\n echo \"ERROR: + failed to raise pull request $output\"\n exit 1\n fi\nfi\n}\n\nsync_spin_to_stash(){\n + \ #setup git configuration using email and username\n setup_git\n #upload spinnaker + configuration to git\n sync_spin_to_git\n #check if custom port is being used + for repo\n if [[ $merge_branch == \"true\" && $target_branch != \"\" && ($git_branch + != $target_branch) ]];then\n if [[ $git_api_url_port != \"\" ]];then\n git_api_url=$git_api_url:$git_api_url_port\n + \ create_pr_stash\n if [[ $auto_merge == \"true\" ]]; then\n approve_pr_stash\n + \ merge_pr_stash\n fi\n else\n create_pr_stash\n if [[ + $auto_merge == \"true\" ]]; then\n approve_pr_stash\n merge_pr_stash\n + \ fi\n fi\n fi\n}\n" +kind: ConfigMap +metadata: + creationTimestamp: null + name: pipe-promot-scripts + namespace: pipe-test diff --git a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml index 6a5fa4bc..031702df 100644 --- a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml @@ -10,5 +10,5 @@ stringData: kind: Secret metadata: name: spin-cli-config - namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} + #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} {{- end }} From c9f849acd75fd10bade279d3247a545201d15808 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 17:55:49 +0530 Subject: [PATCH 04/16] Added git sample --- charts/oes/values-git.yaml | 1089 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1089 insertions(+) create mode 100644 charts/oes/values-git.yaml diff --git a/charts/oes/values-git.yaml b/charts/oes/values-git.yaml new file mode 100644 index 00000000..a1e3c467 --- /dev/null +++ b/charts/oes/values-git.yaml @@ -0,0 +1,1089 @@ +##################################################### +## OpsMx Enterprise for Spinnaker configuration +##################################################### +# Default values for OES chart. +## This is a YAML-formatted file. +## Declare variables to be passed into your templates. + +## Name of the secret for pulling image from docker registry. +## Change it only if you want to create a secret with +## different name before installing oes chart and use it. +imagePullSecret: opsmxdev-secret + +## Quay container image registry credentials to create imagePullSecret +## +imageCredentials: + registry: https://quay.io/ + username: "opsmxpublic+opsmxuser" # Quay username + password: 7AQQKRKTJL8YOQWHF6GFYY6EWQ3INFHL2VMHD0CIK3OYD94I47V0J35INIFZ93V5 + email: info@opsmx.com # email corresponding to quay registry ID + +rbac: + create: true + +#################################################### +## Option to skip installation of OpsMx Spinnaker +## Setting this to true, will ensure custom ubi8 images of Spinnaker +## installSpinnaker and installRedis flags are mutually exclusive +installSpinnaker: true + +#################################################### +## Redis configuration +#################################################### +## Install a seperate Redis for OES if installSpinnaker is set to false +## If Spinnaker is being installed, redis installed by spinnaker chart can +## be re-used +## installSpinnaker and installRedis flags are mutually exclusive +installRedis: false + +## Installation mode +## Available installation modes OES-AP, None +installationMode: OES-AP +oesAutoConfiguration: true # Attempt to configure OES with some basic integrations based on best guess +secretStore: db # Valid values: db, Vault: Used for storing account and integration secrets. Vault, if used, needs to be installed and configured separately +vault: + address: https://server.vaultint.opsmx.net # Vault Address URL + token: 123132 # Vault Root token + +## OES UI, OES Gate, Spinnaker Deck and Spinnaker Gate service type +## +k8sServiceType: ClusterIP + +## Declare all the global variables under this +## Global variables can be accessed across all the charts including sub-charts +global: + ## Set this to false if cert-manager is not installed + ## If cert-manager is installed, an issuer will be created + ## by OES helm chart which generates certs for tls automatically + ## If cert-manager is not installed, specify secrets with + ## certificates under oesUI.tls.secretName & oesGate.tls.secretName + + certManager: + installed: true + self-signed-certs: + enabled: false # At this point this is not supported + + # Below flag is used to setup tls termination at ingress + # when this flag is set to false; spinnaker and oes endpoints + # will be accessible over http instead of https + ssl: + enabled: true + + ## Set to true to expose oes-ui, oes-gate services over ingress + ## + createIngress: true + + # Spinnaker Deck URL configuration; url overwhich spinnaker deck will be accessed + spinDeck: + protocol: https + host: spin.srini2-ryzon7.opsmx.net + #port: 9000 + + ingress: + annotations: + ingress.kubernetes.io/ssl-redirect: "true" + kubernetes.io/ingress.class: nginx + tls: + secretName: deck-authtls + + # Spinnaker Gate URL configuration; url overwhich spinnaker gate will be accessed + spinGate: + protocol: https + host: spin-gate.srini2-ryzon7.opsmx.net + #port: 8084 + + ingress: + annotations: + ingress.kubernetes.io/ssl-redirect: "true" + kubernetes.io/ingress.class: nginx + + tls: + secretName: gate-authtls + + ## Set it to false if own LDAP is to be configured + ## ldap configuration used in oes-gate, oes-platform and spinnaker gate for authentication + ## and authorization + + installOpenLdap: true + ldap: + enabled: true + url: ldap://{{ .Release.Name }}-openldap:389 + managerDn: cn=admin,dc=example,dc=org + managerPassword: opsmxadmin123 + groupSearchBase: ou=groups,dc=example,dc=org + groupSearchFilter: member={0} + groupRoleAttributes: cn + userDnPattern: cn={0},dc=example,dc=org + + +## OES-UI url configuration +oesUI: + protocol: https + host: oes.srini2-ryzon7.opsmx.net + # Use below port when hostname above is an external IP instead of a hostname + #port: 80 + + ingress: + annotations: + kubernetes.io/ingress.class: nginx + + tls: + secretName: oes-ui-authtls + +## OES-Gate url configuration +oesGate: + protocol: https + host: oes-gate.srini2-ryzon7.opsmx.net + # Use below port when hostname above is an external IP instead of a hostname + #port: 8084 + + ingress: + annotations: + kubernetes.io/ingress.class: nginx + + tls: + secretName: oes-gate-authtls + + +############################################################################### +## Details of redis-master image for OES +## +redis: + ## Redis endpoint that is used by oes-gate and oes-platform for caching; + ## Change this to custom URL if installRedis is set to false + ## url: redis://{{ .Release.Name }}-redis-master:6379 + ## + url: redis://:password@{{ .Release.Name }}-redis-master:6379 + password: password + cluster: + enabled: false + +############################################################################### +## +## Values of OES Database +## +db: + ## Set it to false if any external database is to be used + ## + enabled: true + + ## Change the default configuration when above option is set to false + ## Below url and credentials are used by Autopilot & Sapor + url: jdbc:postgresql://oes-db:5432 + username: postgres + password: networks123 + + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-db + tag: v2.0.0 + pullPolicy: IfNotPresent + + ## Strategy to rollout statefulset pods + ## + podManagementPolicy: OrderedReady + + ## Default group to which the default user of a pod belongs + ## + securityContext: + fsGroup: 1000 + + ## storageMountSize is the size with which a PVC is to be created + ## + storageMountSize: 8Gi + + ## storageClass for DB persistent volume claim (PVC) + ## + #storageClassName: default +############################################################################### +## Use this SAPOR GATE Configuration to Enable Basic Authentication for OES SAPOR to communitcate +## with spinnaker instead of x509 +## Values of SAPOR OES Gate +## +saporgate: + ## Image specific details + ## + enabled: false + image: + registry: quay.io/opsmxpublic + repository: ubi8-spin-gate + tag: 1.20.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 500Mi + # cpu: 500m + # limits: + # memory: 1500Mi + # cpu: 1500m + + config: + username: admin # User name with admin permissions and belonging to admin groups defined in platform service + password: saporadmin # Any generic String, need not be the real password + +############################################################################### +## +## Values of OES Autopilot +## +autopilot: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-autopilot + tag: v3.8.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 2Gi + # cpu: 1 + # limits: + # memory: 8Gi + # cpu: 2 + + config: + ## Build Analysis + ## + buildAnalysis: + enabled: false + + ## DB configuration; change it any other external DB is to be used + ## Refer it above, under DB configuration section + ## + + ## SSL configuration + ## + ssl: + enabled: false # Set it to true if SSL is to be enabled + keyStore: keystore.p12 + keyStorePassword: dummypwd # Provide keystore password + keyStoreType: PKCS12 + keyAlias: tomcat + + ## server.host.dns.name is fetched from oesUI.host + ## + ## gate.url is fetched from oesGate.host + ## + +############################################################################### +## +## Values of OES Dashboard +## +dashboard: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-dashboard + tag: v3.8.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 500Mi + # cpu: 500m + # limits: + # memory: 1000Mi + # cpu: 1500m + + config: + ## Installation mode + ## OES supports three modes of installation, + ## Uses the mode configured above, under Installation mode + ## + app: + sync: + enabled: true + ## By default spinnakerLink is {{ .Values.global.spinDeck.protocol }}://{{ .Values.global.spinDeck.host }} + ## If spinnaker is exposed on Load balancer instead of ingress, set this value to external IP of spinnaker UI + #spinnakerLink: http://spinnaker.domain.com + +##################################################### +## +## Values of Forwarder/Controller +## +forwarder: + enabled: true + agent: + image: quay.io/opsmxpublic/forwarder-agent:v20210426T105924 + serviceType: LoadBalancer + # Value is also used in sapor configuration for kubernetes.agent.serverHostName + host: opsmx-controller-controller1 + externalName: controller.srini2-ryzon7.opsmx.net + image: + repository: quay.io/opsmxpublic/forwarder-controller + tag: v20210426T105924 + pullPolicy: IfNotPresent + +############################################################################### +## +## Values of OES Gate +## +gate: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-gate + tag: v3.8.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 500Mi + # cpu: 500m + # limits: + # memory: 1500Mi + # cpu: 1500m + + config: + ## LDAP Configuration + ## Refer above, under LDAP configuration section + ## + + ## Redis Endpoint + ## Uses the URL specified under redis.url above, + ## under Redis Configuration section + ## + + ## Regex of OES-UI URL to allow cross origin requests; this is framed using oesUI.host + + ## Set it to true to disable LDAP authentication and enable file based authentication + ## Reach out over support@opsmx.com for pre-configured user credentials + fileBasedAuthentication: + enabled: false + server: + session: + timeoutInSeconds: 7200 + + ##################################################### + # SAML AUthentication + ##################################################### + # saml.jks and metadata.xml are to be mounted as secrets + # with names oes-saml-jks and oes-saml-metadata resp. + saml: + enabled: false + keyStorePassword: changeit + keyStoreAliasName: saml + issuerId: spind + userSource: gate + + jksSecretName: oes-saml-jks + metadataSecretName: metadata-xml + + ##################################################### + #OAUTH2 Authentication for GitHub + ##################################################### + oauth2: + enabled: false + client: + clientId: #CLIENT_ID + clientSecret: #CLIENT_SECRET_ID + accessTokenUri: https://github.com/login/oauth/access_token + userAuthorizationUri: https://github.com/login/oauth/authorize + scope: user-email + resource: + userInfoUri: https://api.github.com/user + userInfoMapping: + email: email + firstName: firstname + lastName: name + username: login + provider: GITHUB + +############################################################################### +## +## Values of OES Platform +## +platform: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-platform + tag: v3.8.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 500Mi + # cpu: 500m + # limits: + # memory: 1500Mi + # cpu: 1500m + + config: + ## DB configuration; change it any other external DB is to be used + ## Refer it above, under DB configuration section + ## + + ## RBAC Configuration + ## Refer the RBAC create option at the top + ## + + ## Admin Groups available in ldap + ## + adminGroups: admin, Administrators + + ## Source of Users for authorization + ## Supported sources:- ldap, file, gate + userSource: ldap + + ## LDAP Configuration will be used if above userSource is set to ldap + ## Refer above, under LDAP configuration section + ## + + ## Redis Endpoint + ## Uses the URL specified under redis.url above, + ## under Redis Configuration section + ## + + ## List of features to be supported by OES + ## + supportedFeatures: + - deployment_verification + - sapor + - visibility + +############################################################################### +## +## Values of OES Sapor (Security Audit Policy Onboarding & Release) +## +sapor: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-sapor + tag: v3.8.0.1 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 100Mi + # cpu: 500m + # limits: + # memory: 2000Mi + # cpu: 1500m + + config: + ## RBAC Configuration + ## Refer the RBAC create option at the top + ## + + ## Spinnaker Configuration + spinnaker: + ## Spinnaker configuration + ## Necessary details needed by Sapor to + ## integrate with Spinnaker + ## + ## Set it to true if authentication is enabled in Spinnaker + ## + authnEnabled: true + + ## URL of Spinnaker Gate + ## FQDN of spin-gate if Spinnaker is installed in same K8s cluster, else + ## set the url with the external IP address of spin-gate + ## Note: Don't put a trailing / + ## + spinGateURL: http://spin-gate:8084 + + ## By default spinExternalGateURL is {{ .Values.global.spinGate.protocol }}://{{ .Values.global.spinGate.host }} + ## If spinExternalGateURL is an external IP address instead of ingress, Set the external IP address of spin-gate, this is used to redirect to + ## the spinnaker pipelines from OES-UI. + ## Note: Trailing / is not required + ## + #spinExternalGateURL: http://spinnaker-api.domain.com + + ## Spinnaker admin credentials + ## When provided, will override above credentials and operate spinnaker + ## as an admin user + ## + + ## LDAP + ldap: + ldapEnabled: true + ldapUsername: admin + ldapPassword: opsmxadmin123 + ldapAdminLoginEnabled: false + ldapAdminUsername: admin + ldapAdminPassword: admin + + ## X509 + x509: + enabled: false + client: + password: changeit + + ## Set the below field to true if datasource configurations from platform + datasources: + platform: true + + ## Set the below field to true if default java cacert is to be overriden + ## + caCerts: + override: false + +############################################################################### +## +## Values of OES UI +## +ui: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-ui + tag: v3.8.0 + pullPolicy: IfNotPresent + + config: + ## Interval at which UI refreshes application dashboard + setApplicationRefreshInterval: 300000 + +################################################################################### +## +# +## Values of OES Visibility +## +visibility: + ## Image specific details + ## + image: + registry: quay.io/opsmxpublic + repository: ubi8-oes-visibility + tag: v3.8.0 + pullPolicy: IfNotPresent + + resources: {} + # requests: + # memory: 500Mi + # cpu: 500m + # limits: + # memory: 1000Mi + # cpu: 1500m + + config: + ## DB configuration; change it any other external DB is to be used + ## Refer it above, under DB configuration section + ## + + ## Autopilot integration options + configuredConnectors: JIRA,GIT,AUTOPILOT,SONARQUBE,JENKINS,AQUAWAVE + + ## Logger level + logLevel: ERROR + + + +############################################################################### +## This option enables OES to be configured automatically +## Load Balancer IPs will be automatically replaced in the +## configuration files of oes-gate, oes-ui +autoConfiguration: + # Set it to false if OES is being installed on restricted evnironment; + # Autoconfiguration assumes Load Balancer is available for oes-gate, oes-ui + # and spind-deck and configures accordingly + enabled: false + + initContainer: + # Image for init container to automatically configure oes components + # during startup + image: quay.io/opsmxpublic/oes-init:v3 + pullPolicy: IfNotPresent + + # Max time(in secs) that an init container of oes-ui should wait + # to fetch External Load Balancer IP of oes-gate and vice versa + externalIpCheckDelay: 180 + +##################################################### +## OPA configuration +##################################################### +opa: + enabled: true + image: + repository: openpolicyagent/opa + tag: edge-rootless + pullPolicy: IfNotPresent + +#################################################### +## OPENLDAP configuration +#################################################### + +## OpenLDAP custom configuration; will override default configuration of +## openldap helm chart +## +openldap: + # Password for the admin user; by default it is set to admin + adminPassword: opsmxadmin123 + configPassword: opsmxconfig123 + omitClusterIP: true + persistence: + enabled: false + env: + LDAP_REMOVE_CONFIG_AFTER_SETUP: "false" + + customLdifFiles: + 01-memberof.ldif: |- + dn: cn=module,cn=config + cn: module + objectClass: olcModuleList + olcModuleLoad: memberof.la + olcModulePath: /usr/lib/ldap + + dn: olcOverlay={0}memberof,olcDatabase={1}hdb,cn=config + objectClass: olcConfig + objectClass: olcMemberOf + objectClass: olcOverlayConfig + objectClass: top + olcOverlay: memberof + olcMemberOfDangling: ignore + olcMemberOfRefInt: TRUE + olcMemberOfGroupOC: groupOfNames + olcMemberOfMemberAD: member + olcMemberOfMemberOfAD: memberOf + 02-refint1.ldif: |- + dn: cn=module{1},cn=config + changetype: modify + add: olcmoduleload + olcmoduleload: refint.la + 03-refint2.ldif: |- + dn: olcOverlay={1}refint,olcDatabase={1}hdb,cn=config + objectClass: olcConfig + objectClass: olcOverlayConfig + objectClass: olcRefintConfig + objectClass: top + olcOverlay: {1}refint + olcRefintAttribute: memberof member manager owner + 04-add_ou.ldif: |- + dn: ou=groups,dc=example,dc=org + objectClass: organizationalUnit + ou: Groups + 05-admin.ldif: |- + dn: cn=admin,ou=groups,dc=example,dc=org + objectClass: groupofnames + cn: admin + description: read write and execute group + member: cn=admin,dc=example,dc=org + 06-developer.ldif: |- + dn: cn=developers,ou=groups,dc=example,dc=org + objectClass: groupofnames + cn: developers + description: read only users + member: cn=admin,dc=example,dc=org + member: cn=developer,dc=example,dc=org + 07-qa.ldif: |- + dn: cn=QA,ou=groups,dc=example,dc=org + objectClass: groupofnames + cn: QA + description: read only users + member: cn=admin,dc=example,dc=org + member: cn=qa,dc=example,dc=org + 08-manager.ldif: |- + dn: cn=managers,ou=groups,dc=example,dc=org + objectClass: groupofnames + cn: managers + description: read and execute group + member: cn=admin,dc=example,dc=org + member: cn=manager,dc=example,dc=org + 09-IT-manager.ldif: |- + dn: cn=ITManagers,ou=groups,dc=example,dc=org + objectClass: groupofnames + cn: ITManagers + description: read and execute group + member: cn=admin,dc=example,dc=org + member: cn=ITManager,dc=example,dc=org + +## ldap configuration used in oes-gate, oes-platform and spinnaker gate for authentication +## and authorization + +##################################################### +# Centralized Logging Configuration +##################################################### +enableCentralLogging: false +elasticsearch: + replicas: 1 + minimumMasterNodes: 1 + resources: + requests: + cpu: "100m" + memory: "1Gi" + +kibana: + service: + type: LoadBalancer + resources: + requests: + cpu: "100m" + memory: "250Mi" + lifecycle: + postStart: + exec: + command: + - bash + - -c + - > + until curl localhost:5601; do echo "Waiting for Kibana to be available..."; sleep 5; done; + until curl elasticsearch-master:9200; do echo "Waiting for Elasticsearch to be available..."; sleep 5; done; + sleep 60; + curl https://raw.githubusercontent.com/OpsMx/enterprise-spinnaker/master/scripts/kibana/kibana_objects.ndjson > /tmp/kibana_objects.ndjson; + curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/tmp/kibana_objects.ndjson 2>&1 1> /tmp/postStart.out; + +##################################################### +# Spinnaker instance configuration +##################################################### +spinnaker: + halyard: + spinnakerVersion: 1.24.4 + image: + repository: quay.io/opsmxpublic/ubi8-spin-halyard + tag: opsmx-1.40.0 + # Set to false to disable persistence data volume for halyard + persistence: + enabled: true + # Provide a config map with Hal commands that will be run the core config (storage) + # The config map should contain a script in the config.sh key + additionalScripts: + enabled: false + configMapName: my-halyard-config + configMapKey: config.sh + # If you'd rather do an inline script, set create to true and put the content in the data dict like you would a configmap + # The content will be passed through `tpl`, so value interpolation is supported. + create: false + data: {} + additionalSecrets: + create: false + data: {} + ## Uncomment if you want to use a pre-created secret rather than feeding data in via helm. + # name: + additionalConfigMaps: + create: false + data: {} + ## Uncomment if you want to use a pre-created ConfigMap rather than feeding data in via helm. + # name: + additionalProfileConfigMaps: + data: + ## if you're running spinnaker behind a reverse proxy such as a GCE ingress + ## you may need the following profile settings for the gate profile. + ## see https://github.com/spinnaker/spinnaker/issues/1630 + ## otherwise its harmless and will likely become default behavior in the future + ## According to the linked github issue. + gate-local.yml: + server: + tomcat: + protocolHeader: X-Forwarded-Proto + remoteIpHeader: X-Forwarded-For + internalProxies: .* + httpsServerPort: X-Forwarded-Port + + ##If opa is installed and enabled and spinnaker is installed, + ##Then you can enable policy in spinnaker through front50-local yaml. + ##If you don't want to configure make it as false or + ## If you have your different opa server you can mention that url here + front50-local.yml: + policy: + opa: + enabled: true + url: http://oes-sapor.{{ .Release.Namespace }}:8085 + + ## Define custom settings for Spinnaker services. Read more for details: + ## https://www.spinnaker.io/reference/halyard/custom/#custom-service-settings + ## You can use it to add annotations for pods, override the image, etc. + additionalServiceSettings: + ## artifactId to override Spinnaker components images with OpsMx custom images + gate.yml: + healthEndpoint: /health + kubernetes: + useExecHealthCheck: false + artifactId: quay.io/opsmxpublic/ubi8-spin-gate:1.20.0 + deck.yml: + artifactId: quay.io/opsmxpublic/ubi8-oes-deck:3.5.1 + clouddriver.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 + clouddriver-caching.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 + clouddriver-rw.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 + clouddriver-ro.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 + clouddriver-ro-deck.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 + echo.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 + echo-scheduler.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 + echo-worker.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 + fiat.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-fiat:1.14.1 + front50.yml: + artifactId: quay.io/opsmxpublic/ubi8-oes-front50:0.26.1 + igor.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-igor:1.14.0 + kayenta.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-kayenta:0.19.0 + orca.yml: + artifactId: quay.io/opsmxpublic/ubi8-oes-orca:2.18.1 + rosco.yml: + artifactId: quay.io/opsmxpublic/ubi8-spin-rosco:0.23.0 + ## Uncomment if you want to add extra commands to the init script + ## run by the init container before halyard is started. + ## The content will be passed through `tpl`, so value interpolation is supported. + # additionalInitScript: |- + + ## Uncomment if you want to add annotations on halyard and install-using-hal pods + # annotations: + # iam.amazonaws.com/role: + + ## Uncomment the following resources definitions to control the cpu and memory + # resources allocated for the halyard pod + resources: + {} + # requests: + # memory: "1Gi" + # cpu: "100m" + # limits: + # memory: "2Gi" + # cpu: "200m" + + ## Uncomment if you want to set environment variables on the Halyard pod. + # env: + # - name: JAVA_OPTS + # value: -Dhttp.proxyHost=proxy.example.com + customCerts: + ## Enable to override the default cacerts with your own one + enabled: false + secretName: custom-cacerts + + # Define which registries and repositories you want available in your + # Spinnaker pipeline definitions + # For more info visit: + # https://www.spinnaker.io/setup/providers/docker-registry/ + + # Configure your Docker registries here + dockerRegistries: + - name: dockerhub + address: index.docker.io + repositories: + - library/alpine + - library/ubuntu + - library/centos + - library/nginx + # - name: gcr + # address: https://gcr.io + # username: _json_key + # password: '' + # email: 1234@5678.com + + # If you don't want to put your passwords into a values file + # you can use a pre-created secret instead of putting passwords + # (specify secret name in below `dockerRegistryAccountSecret`) + # per account above with data in the format: + # : + + # dockerRegistryAccountSecret: myregistry-secrets + + kubeConfig: + # Use this when you want to register arbitrary clusters with Spinnaker + # Upload your ~/kube/.config to a secret + enabled: false + secretName: my-kubeconfig + secretKey: config + # List of contexts from the kubeconfig to make available to Spinnaker + contexts: + - default + deploymentContext: default + omittedNameSpaces: + - kube-system + - kube-public + onlySpinnakerManaged: + enabled: true + + # spinnakerFeatureFlags is a list of Spinnaker feature flags to enable + # Ref: https://www.spinnaker.io/reference/halyard/commands/#hal-config-features-edit + # spinnakerFeatureFlags: + # - artifacts + # - pipeline-templates + spinnakerFeatureFlags: [] + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + # nodeSelector to provide to each of the Spinnaker components + nodeSelector: {} + + # Redis password to use for the in-cluster redis service + # Enable redis to use in-cluster redis + redis: + enabled: true + # External Redis option will be enabled if in-cluster redis is disabled + external: + host: "" + port: 6379 + # password: "" + password: password + nodeSelector: {} + cluster: + enabled: false + + ## Redis config file + ## ref: https://redis.io/topics/config + ## + configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly no + # Disable RDB persistence, AOF persistence already enabled. + save 60 1000 + + # Use ubi8 custom images + #image: + # registry: docker.io + # repository: devopsmx/ubi8-oes-redis + # tag: 4.0.14 + # Uncomment if you don't want to create a PVC for redis + # master: + # persistence: + # enabled: false + + # Minio access/secret keys for the in-cluster S3 usage + # Minio is not exposed publically + minio: + enabled: true + imageTag: RELEASE.2019-02-13T19-48-27Z + serviceType: ClusterIP + accessKey: spinnakeradmin + secretKey: spinnakeradmin + bucket: "spinnaker" + nodeSelector: {} + persistence: + enabled: true + size: 10Gi + # Use ubi8 custom images + #image: + # repository: devopsmx/ubi8-oes-minio + # tag: RELEASE.2019-09-18T21-55-05Z + # pullPolicy: IfNotPresent + + # Google Cloud Storage + gcs: + enabled: false + project: my-project-name + bucket: "" + ## if jsonKey is set, will create a secret containing it + jsonKey: "" + ## override the name of the secret to use for jsonKey, if `jsonKey` + ## is empty, it will not create a secret assuming you are creating one + ## external to the chart. the key for that secret should be `key.json`. + secretName: + + # AWS Simple Storage Service + s3: + enabled: false + bucket: "" + # rootFolder: "front50" + # region: "us-east-1" + # endpoint: "" + # accessKey: "" + # secretKey: "" + + # Azure Storage Account + azs: + enabled: false + # storageAccountName: "" + # accessKey: "" + # containerName: "spinnaker" + + rbac: + # Specifies whether RBAC resources should be created + create: true + + serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccounts to use. + # If left blank it is auto-generated from the fullname of the release + halyardName: + spinnakerName: + securityContext: + # Specifies permissions to write for user/group + runAsUser: 1000 + fsGroup: 1000 + + ## Option to enable HA in Spinnaker; Enabled by default + enableHA: true + + ## Centralized Monitoring enable for Prometheus + enableCentralMonitoring: false + + # Initialize gitOps style Halyard + gitopsHalyard: + enabled: true + repo: + type: git # git, S3, stash(bitbucket server) use a different sample values file for s3/bitbuck-stash + ## Configure below fields only if repo type is git/stash. Skip to s3 section if type is s3 + baseUrlHostName: github.com # Specify it if git url is something other than github.com + organization: ksrinimba + # Specify project name only if repo is under a project + projectName: "" + repository: ryzon7-gitops + dynamicAccRepository: ryzon7-gitops + halConfigPath: / # Any other value is currently not supported + username: ksrinimba + token: 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 + secretName: opsmx-gitops-auth + # Promote applications and pipelines from one environment to another or take backup + pipelinePromotion: # GitHub only, Not supportd on S3 or Stash + enabled: true + type: github + organization: ksrinimba # Also called "project" in some repos + repository: ryzon7-gitops # bucket name in case of S3 + rootFolder: pipeline-promotion/ + ##### ONLY In case of S3 + AWS_ACCESS_KEY_ID: DFGHJ + AWS_SECRET_ACCESS_KEY: 45678kk + ##### S3 config for pipelinePromotion is complete + ##### For non-S3 repos + baseUrl: "github.com" # "git_url" + username: ksrinimba + token: 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 + branch: "kishore" + usermail: ksrinimba@gmail.com + #password="Krrish#438" ### Token is preferred, Password also might work, try your luck + # Instead of username, token/password, sshkey can be provided + #secret_sshkey="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC83mRU9Euwx/h547gNjMsWyA86MybQGRx6p9OyNp98RxLXjrG2B5D79F73Cp6eoA2I/zw4dxtPeIll88Yd0z4m0rp8p8FSmCrxOPauIzhPj5e4yRcEfLJ8LA3Q7Xl2Vfg1rUQ3PdaCpr4OZvy8i3kjGumoZ0rVvGW0LHMrDGXXz3FGeLD5dT/lpHJo1kbelQ/28K4xqdZQOOi0Z2oQemyiKdVifZBQR2NFaD72fQD5t0FbJLtgfm3Ko3sSFa00nITIfyUxCCRQbmd+Y+Dnl1dY9FS31sXXvxzJr62cn911igMbHBXvX77vQHoX7Bc9a0rrfnsulyzb5d11Iw0ywHcx kishore@kishore-Latitude-3510" + #API + #api_url="https://api.bitbucket.org/2.0/repositories/" # bitbucket + apiUrl: "https://api.github.com/repos" # guthub + #api_url="https://api.gitlab.com/repos" # gitLAB + #api_url="https://bb.opsmx.com/api" # bitbucket-server(stash) + + createPR: true + autoApprovePR: true + targetBranch: master + approvingUser: "ksrinimba" ### user who is going to auto-merge + approverToken: "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" ## Token for the user to auto-merge + #approver_password: "" ### Token is preferred, Password also might work, try it + #approver_secret_sshkey= ??? + ## x509 authentication for Spinnaker Gate + gatex509: + enabled: false + host: spingate-x509.domain.com + # Max time(in secs) that an init container of halyard should wait + # to fetch External Load Balancer IP of spin-deck and spin-gate + spinnakerLBCheckDelay: 1 # This is only used if not using ingress + mTLS: + enabled: false # Enable mTLS for Spinnaker Services and SSL for Deck and Gate + + ## Auth mechanism and credentials to be used by spin-cli for creating the sample pipelines + ## Here basic ldap auth is used by default; everything under spinCli will be pasted in ~/.spin/config + spinCli: + gate: + endpoint: http://spin-gate:8084 # Should same as the spin-gate URL + auth: + enabled: true + basic: + username: admin # Please updated as required + password: opsmxadmin123 # Please updated as required From 72ba4a9500f5cebcb36de47bd30558632660a4cc Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 21:19:04 +0530 Subject: [PATCH 05/16] Removeing temp file --- charts/oes/values-git.yaml | 1089 ------------------------------------ 1 file changed, 1089 deletions(-) delete mode 100644 charts/oes/values-git.yaml diff --git a/charts/oes/values-git.yaml b/charts/oes/values-git.yaml deleted file mode 100644 index a1e3c467..00000000 --- a/charts/oes/values-git.yaml +++ /dev/null @@ -1,1089 +0,0 @@ -##################################################### -## OpsMx Enterprise for Spinnaker configuration -##################################################### -# Default values for OES chart. -## This is a YAML-formatted file. -## Declare variables to be passed into your templates. - -## Name of the secret for pulling image from docker registry. -## Change it only if you want to create a secret with -## different name before installing oes chart and use it. -imagePullSecret: opsmxdev-secret - -## Quay container image registry credentials to create imagePullSecret -## -imageCredentials: - registry: https://quay.io/ - username: "opsmxpublic+opsmxuser" # Quay username - password: 7AQQKRKTJL8YOQWHF6GFYY6EWQ3INFHL2VMHD0CIK3OYD94I47V0J35INIFZ93V5 - email: info@opsmx.com # email corresponding to quay registry ID - -rbac: - create: true - -#################################################### -## Option to skip installation of OpsMx Spinnaker -## Setting this to true, will ensure custom ubi8 images of Spinnaker -## installSpinnaker and installRedis flags are mutually exclusive -installSpinnaker: true - -#################################################### -## Redis configuration -#################################################### -## Install a seperate Redis for OES if installSpinnaker is set to false -## If Spinnaker is being installed, redis installed by spinnaker chart can -## be re-used -## installSpinnaker and installRedis flags are mutually exclusive -installRedis: false - -## Installation mode -## Available installation modes OES-AP, None -installationMode: OES-AP -oesAutoConfiguration: true # Attempt to configure OES with some basic integrations based on best guess -secretStore: db # Valid values: db, Vault: Used for storing account and integration secrets. Vault, if used, needs to be installed and configured separately -vault: - address: https://server.vaultint.opsmx.net # Vault Address URL - token: 123132 # Vault Root token - -## OES UI, OES Gate, Spinnaker Deck and Spinnaker Gate service type -## -k8sServiceType: ClusterIP - -## Declare all the global variables under this -## Global variables can be accessed across all the charts including sub-charts -global: - ## Set this to false if cert-manager is not installed - ## If cert-manager is installed, an issuer will be created - ## by OES helm chart which generates certs for tls automatically - ## If cert-manager is not installed, specify secrets with - ## certificates under oesUI.tls.secretName & oesGate.tls.secretName - - certManager: - installed: true - self-signed-certs: - enabled: false # At this point this is not supported - - # Below flag is used to setup tls termination at ingress - # when this flag is set to false; spinnaker and oes endpoints - # will be accessible over http instead of https - ssl: - enabled: true - - ## Set to true to expose oes-ui, oes-gate services over ingress - ## - createIngress: true - - # Spinnaker Deck URL configuration; url overwhich spinnaker deck will be accessed - spinDeck: - protocol: https - host: spin.srini2-ryzon7.opsmx.net - #port: 9000 - - ingress: - annotations: - ingress.kubernetes.io/ssl-redirect: "true" - kubernetes.io/ingress.class: nginx - tls: - secretName: deck-authtls - - # Spinnaker Gate URL configuration; url overwhich spinnaker gate will be accessed - spinGate: - protocol: https - host: spin-gate.srini2-ryzon7.opsmx.net - #port: 8084 - - ingress: - annotations: - ingress.kubernetes.io/ssl-redirect: "true" - kubernetes.io/ingress.class: nginx - - tls: - secretName: gate-authtls - - ## Set it to false if own LDAP is to be configured - ## ldap configuration used in oes-gate, oes-platform and spinnaker gate for authentication - ## and authorization - - installOpenLdap: true - ldap: - enabled: true - url: ldap://{{ .Release.Name }}-openldap:389 - managerDn: cn=admin,dc=example,dc=org - managerPassword: opsmxadmin123 - groupSearchBase: ou=groups,dc=example,dc=org - groupSearchFilter: member={0} - groupRoleAttributes: cn - userDnPattern: cn={0},dc=example,dc=org - - -## OES-UI url configuration -oesUI: - protocol: https - host: oes.srini2-ryzon7.opsmx.net - # Use below port when hostname above is an external IP instead of a hostname - #port: 80 - - ingress: - annotations: - kubernetes.io/ingress.class: nginx - - tls: - secretName: oes-ui-authtls - -## OES-Gate url configuration -oesGate: - protocol: https - host: oes-gate.srini2-ryzon7.opsmx.net - # Use below port when hostname above is an external IP instead of a hostname - #port: 8084 - - ingress: - annotations: - kubernetes.io/ingress.class: nginx - - tls: - secretName: oes-gate-authtls - - -############################################################################### -## Details of redis-master image for OES -## -redis: - ## Redis endpoint that is used by oes-gate and oes-platform for caching; - ## Change this to custom URL if installRedis is set to false - ## url: redis://{{ .Release.Name }}-redis-master:6379 - ## - url: redis://:password@{{ .Release.Name }}-redis-master:6379 - password: password - cluster: - enabled: false - -############################################################################### -## -## Values of OES Database -## -db: - ## Set it to false if any external database is to be used - ## - enabled: true - - ## Change the default configuration when above option is set to false - ## Below url and credentials are used by Autopilot & Sapor - url: jdbc:postgresql://oes-db:5432 - username: postgres - password: networks123 - - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-db - tag: v2.0.0 - pullPolicy: IfNotPresent - - ## Strategy to rollout statefulset pods - ## - podManagementPolicy: OrderedReady - - ## Default group to which the default user of a pod belongs - ## - securityContext: - fsGroup: 1000 - - ## storageMountSize is the size with which a PVC is to be created - ## - storageMountSize: 8Gi - - ## storageClass for DB persistent volume claim (PVC) - ## - #storageClassName: default -############################################################################### -## Use this SAPOR GATE Configuration to Enable Basic Authentication for OES SAPOR to communitcate -## with spinnaker instead of x509 -## Values of SAPOR OES Gate -## -saporgate: - ## Image specific details - ## - enabled: false - image: - registry: quay.io/opsmxpublic - repository: ubi8-spin-gate - tag: 1.20.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 500Mi - # cpu: 500m - # limits: - # memory: 1500Mi - # cpu: 1500m - - config: - username: admin # User name with admin permissions and belonging to admin groups defined in platform service - password: saporadmin # Any generic String, need not be the real password - -############################################################################### -## -## Values of OES Autopilot -## -autopilot: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-autopilot - tag: v3.8.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 2Gi - # cpu: 1 - # limits: - # memory: 8Gi - # cpu: 2 - - config: - ## Build Analysis - ## - buildAnalysis: - enabled: false - - ## DB configuration; change it any other external DB is to be used - ## Refer it above, under DB configuration section - ## - - ## SSL configuration - ## - ssl: - enabled: false # Set it to true if SSL is to be enabled - keyStore: keystore.p12 - keyStorePassword: dummypwd # Provide keystore password - keyStoreType: PKCS12 - keyAlias: tomcat - - ## server.host.dns.name is fetched from oesUI.host - ## - ## gate.url is fetched from oesGate.host - ## - -############################################################################### -## -## Values of OES Dashboard -## -dashboard: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-dashboard - tag: v3.8.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 500Mi - # cpu: 500m - # limits: - # memory: 1000Mi - # cpu: 1500m - - config: - ## Installation mode - ## OES supports three modes of installation, - ## Uses the mode configured above, under Installation mode - ## - app: - sync: - enabled: true - ## By default spinnakerLink is {{ .Values.global.spinDeck.protocol }}://{{ .Values.global.spinDeck.host }} - ## If spinnaker is exposed on Load balancer instead of ingress, set this value to external IP of spinnaker UI - #spinnakerLink: http://spinnaker.domain.com - -##################################################### -## -## Values of Forwarder/Controller -## -forwarder: - enabled: true - agent: - image: quay.io/opsmxpublic/forwarder-agent:v20210426T105924 - serviceType: LoadBalancer - # Value is also used in sapor configuration for kubernetes.agent.serverHostName - host: opsmx-controller-controller1 - externalName: controller.srini2-ryzon7.opsmx.net - image: - repository: quay.io/opsmxpublic/forwarder-controller - tag: v20210426T105924 - pullPolicy: IfNotPresent - -############################################################################### -## -## Values of OES Gate -## -gate: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-gate - tag: v3.8.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 500Mi - # cpu: 500m - # limits: - # memory: 1500Mi - # cpu: 1500m - - config: - ## LDAP Configuration - ## Refer above, under LDAP configuration section - ## - - ## Redis Endpoint - ## Uses the URL specified under redis.url above, - ## under Redis Configuration section - ## - - ## Regex of OES-UI URL to allow cross origin requests; this is framed using oesUI.host - - ## Set it to true to disable LDAP authentication and enable file based authentication - ## Reach out over support@opsmx.com for pre-configured user credentials - fileBasedAuthentication: - enabled: false - server: - session: - timeoutInSeconds: 7200 - - ##################################################### - # SAML AUthentication - ##################################################### - # saml.jks and metadata.xml are to be mounted as secrets - # with names oes-saml-jks and oes-saml-metadata resp. - saml: - enabled: false - keyStorePassword: changeit - keyStoreAliasName: saml - issuerId: spind - userSource: gate - - jksSecretName: oes-saml-jks - metadataSecretName: metadata-xml - - ##################################################### - #OAUTH2 Authentication for GitHub - ##################################################### - oauth2: - enabled: false - client: - clientId: #CLIENT_ID - clientSecret: #CLIENT_SECRET_ID - accessTokenUri: https://github.com/login/oauth/access_token - userAuthorizationUri: https://github.com/login/oauth/authorize - scope: user-email - resource: - userInfoUri: https://api.github.com/user - userInfoMapping: - email: email - firstName: firstname - lastName: name - username: login - provider: GITHUB - -############################################################################### -## -## Values of OES Platform -## -platform: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-platform - tag: v3.8.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 500Mi - # cpu: 500m - # limits: - # memory: 1500Mi - # cpu: 1500m - - config: - ## DB configuration; change it any other external DB is to be used - ## Refer it above, under DB configuration section - ## - - ## RBAC Configuration - ## Refer the RBAC create option at the top - ## - - ## Admin Groups available in ldap - ## - adminGroups: admin, Administrators - - ## Source of Users for authorization - ## Supported sources:- ldap, file, gate - userSource: ldap - - ## LDAP Configuration will be used if above userSource is set to ldap - ## Refer above, under LDAP configuration section - ## - - ## Redis Endpoint - ## Uses the URL specified under redis.url above, - ## under Redis Configuration section - ## - - ## List of features to be supported by OES - ## - supportedFeatures: - - deployment_verification - - sapor - - visibility - -############################################################################### -## -## Values of OES Sapor (Security Audit Policy Onboarding & Release) -## -sapor: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-sapor - tag: v3.8.0.1 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 100Mi - # cpu: 500m - # limits: - # memory: 2000Mi - # cpu: 1500m - - config: - ## RBAC Configuration - ## Refer the RBAC create option at the top - ## - - ## Spinnaker Configuration - spinnaker: - ## Spinnaker configuration - ## Necessary details needed by Sapor to - ## integrate with Spinnaker - ## - ## Set it to true if authentication is enabled in Spinnaker - ## - authnEnabled: true - - ## URL of Spinnaker Gate - ## FQDN of spin-gate if Spinnaker is installed in same K8s cluster, else - ## set the url with the external IP address of spin-gate - ## Note: Don't put a trailing / - ## - spinGateURL: http://spin-gate:8084 - - ## By default spinExternalGateURL is {{ .Values.global.spinGate.protocol }}://{{ .Values.global.spinGate.host }} - ## If spinExternalGateURL is an external IP address instead of ingress, Set the external IP address of spin-gate, this is used to redirect to - ## the spinnaker pipelines from OES-UI. - ## Note: Trailing / is not required - ## - #spinExternalGateURL: http://spinnaker-api.domain.com - - ## Spinnaker admin credentials - ## When provided, will override above credentials and operate spinnaker - ## as an admin user - ## - - ## LDAP - ldap: - ldapEnabled: true - ldapUsername: admin - ldapPassword: opsmxadmin123 - ldapAdminLoginEnabled: false - ldapAdminUsername: admin - ldapAdminPassword: admin - - ## X509 - x509: - enabled: false - client: - password: changeit - - ## Set the below field to true if datasource configurations from platform - datasources: - platform: true - - ## Set the below field to true if default java cacert is to be overriden - ## - caCerts: - override: false - -############################################################################### -## -## Values of OES UI -## -ui: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-ui - tag: v3.8.0 - pullPolicy: IfNotPresent - - config: - ## Interval at which UI refreshes application dashboard - setApplicationRefreshInterval: 300000 - -################################################################################### -## -# -## Values of OES Visibility -## -visibility: - ## Image specific details - ## - image: - registry: quay.io/opsmxpublic - repository: ubi8-oes-visibility - tag: v3.8.0 - pullPolicy: IfNotPresent - - resources: {} - # requests: - # memory: 500Mi - # cpu: 500m - # limits: - # memory: 1000Mi - # cpu: 1500m - - config: - ## DB configuration; change it any other external DB is to be used - ## Refer it above, under DB configuration section - ## - - ## Autopilot integration options - configuredConnectors: JIRA,GIT,AUTOPILOT,SONARQUBE,JENKINS,AQUAWAVE - - ## Logger level - logLevel: ERROR - - - -############################################################################### -## This option enables OES to be configured automatically -## Load Balancer IPs will be automatically replaced in the -## configuration files of oes-gate, oes-ui -autoConfiguration: - # Set it to false if OES is being installed on restricted evnironment; - # Autoconfiguration assumes Load Balancer is available for oes-gate, oes-ui - # and spind-deck and configures accordingly - enabled: false - - initContainer: - # Image for init container to automatically configure oes components - # during startup - image: quay.io/opsmxpublic/oes-init:v3 - pullPolicy: IfNotPresent - - # Max time(in secs) that an init container of oes-ui should wait - # to fetch External Load Balancer IP of oes-gate and vice versa - externalIpCheckDelay: 180 - -##################################################### -## OPA configuration -##################################################### -opa: - enabled: true - image: - repository: openpolicyagent/opa - tag: edge-rootless - pullPolicy: IfNotPresent - -#################################################### -## OPENLDAP configuration -#################################################### - -## OpenLDAP custom configuration; will override default configuration of -## openldap helm chart -## -openldap: - # Password for the admin user; by default it is set to admin - adminPassword: opsmxadmin123 - configPassword: opsmxconfig123 - omitClusterIP: true - persistence: - enabled: false - env: - LDAP_REMOVE_CONFIG_AFTER_SETUP: "false" - - customLdifFiles: - 01-memberof.ldif: |- - dn: cn=module,cn=config - cn: module - objectClass: olcModuleList - olcModuleLoad: memberof.la - olcModulePath: /usr/lib/ldap - - dn: olcOverlay={0}memberof,olcDatabase={1}hdb,cn=config - objectClass: olcConfig - objectClass: olcMemberOf - objectClass: olcOverlayConfig - objectClass: top - olcOverlay: memberof - olcMemberOfDangling: ignore - olcMemberOfRefInt: TRUE - olcMemberOfGroupOC: groupOfNames - olcMemberOfMemberAD: member - olcMemberOfMemberOfAD: memberOf - 02-refint1.ldif: |- - dn: cn=module{1},cn=config - changetype: modify - add: olcmoduleload - olcmoduleload: refint.la - 03-refint2.ldif: |- - dn: olcOverlay={1}refint,olcDatabase={1}hdb,cn=config - objectClass: olcConfig - objectClass: olcOverlayConfig - objectClass: olcRefintConfig - objectClass: top - olcOverlay: {1}refint - olcRefintAttribute: memberof member manager owner - 04-add_ou.ldif: |- - dn: ou=groups,dc=example,dc=org - objectClass: organizationalUnit - ou: Groups - 05-admin.ldif: |- - dn: cn=admin,ou=groups,dc=example,dc=org - objectClass: groupofnames - cn: admin - description: read write and execute group - member: cn=admin,dc=example,dc=org - 06-developer.ldif: |- - dn: cn=developers,ou=groups,dc=example,dc=org - objectClass: groupofnames - cn: developers - description: read only users - member: cn=admin,dc=example,dc=org - member: cn=developer,dc=example,dc=org - 07-qa.ldif: |- - dn: cn=QA,ou=groups,dc=example,dc=org - objectClass: groupofnames - cn: QA - description: read only users - member: cn=admin,dc=example,dc=org - member: cn=qa,dc=example,dc=org - 08-manager.ldif: |- - dn: cn=managers,ou=groups,dc=example,dc=org - objectClass: groupofnames - cn: managers - description: read and execute group - member: cn=admin,dc=example,dc=org - member: cn=manager,dc=example,dc=org - 09-IT-manager.ldif: |- - dn: cn=ITManagers,ou=groups,dc=example,dc=org - objectClass: groupofnames - cn: ITManagers - description: read and execute group - member: cn=admin,dc=example,dc=org - member: cn=ITManager,dc=example,dc=org - -## ldap configuration used in oes-gate, oes-platform and spinnaker gate for authentication -## and authorization - -##################################################### -# Centralized Logging Configuration -##################################################### -enableCentralLogging: false -elasticsearch: - replicas: 1 - minimumMasterNodes: 1 - resources: - requests: - cpu: "100m" - memory: "1Gi" - -kibana: - service: - type: LoadBalancer - resources: - requests: - cpu: "100m" - memory: "250Mi" - lifecycle: - postStart: - exec: - command: - - bash - - -c - - > - until curl localhost:5601; do echo "Waiting for Kibana to be available..."; sleep 5; done; - until curl elasticsearch-master:9200; do echo "Waiting for Elasticsearch to be available..."; sleep 5; done; - sleep 60; - curl https://raw.githubusercontent.com/OpsMx/enterprise-spinnaker/master/scripts/kibana/kibana_objects.ndjson > /tmp/kibana_objects.ndjson; - curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/tmp/kibana_objects.ndjson 2>&1 1> /tmp/postStart.out; - -##################################################### -# Spinnaker instance configuration -##################################################### -spinnaker: - halyard: - spinnakerVersion: 1.24.4 - image: - repository: quay.io/opsmxpublic/ubi8-spin-halyard - tag: opsmx-1.40.0 - # Set to false to disable persistence data volume for halyard - persistence: - enabled: true - # Provide a config map with Hal commands that will be run the core config (storage) - # The config map should contain a script in the config.sh key - additionalScripts: - enabled: false - configMapName: my-halyard-config - configMapKey: config.sh - # If you'd rather do an inline script, set create to true and put the content in the data dict like you would a configmap - # The content will be passed through `tpl`, so value interpolation is supported. - create: false - data: {} - additionalSecrets: - create: false - data: {} - ## Uncomment if you want to use a pre-created secret rather than feeding data in via helm. - # name: - additionalConfigMaps: - create: false - data: {} - ## Uncomment if you want to use a pre-created ConfigMap rather than feeding data in via helm. - # name: - additionalProfileConfigMaps: - data: - ## if you're running spinnaker behind a reverse proxy such as a GCE ingress - ## you may need the following profile settings for the gate profile. - ## see https://github.com/spinnaker/spinnaker/issues/1630 - ## otherwise its harmless and will likely become default behavior in the future - ## According to the linked github issue. - gate-local.yml: - server: - tomcat: - protocolHeader: X-Forwarded-Proto - remoteIpHeader: X-Forwarded-For - internalProxies: .* - httpsServerPort: X-Forwarded-Port - - ##If opa is installed and enabled and spinnaker is installed, - ##Then you can enable policy in spinnaker through front50-local yaml. - ##If you don't want to configure make it as false or - ## If you have your different opa server you can mention that url here - front50-local.yml: - policy: - opa: - enabled: true - url: http://oes-sapor.{{ .Release.Namespace }}:8085 - - ## Define custom settings for Spinnaker services. Read more for details: - ## https://www.spinnaker.io/reference/halyard/custom/#custom-service-settings - ## You can use it to add annotations for pods, override the image, etc. - additionalServiceSettings: - ## artifactId to override Spinnaker components images with OpsMx custom images - gate.yml: - healthEndpoint: /health - kubernetes: - useExecHealthCheck: false - artifactId: quay.io/opsmxpublic/ubi8-spin-gate:1.20.0 - deck.yml: - artifactId: quay.io/opsmxpublic/ubi8-oes-deck:3.5.1 - clouddriver.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 - clouddriver-caching.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 - clouddriver-rw.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 - clouddriver-ro.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 - clouddriver-ro-deck.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-clouddriver:7.2.2 - echo.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 - echo-scheduler.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 - echo-worker.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-echo:2.15.2 - fiat.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-fiat:1.14.1 - front50.yml: - artifactId: quay.io/opsmxpublic/ubi8-oes-front50:0.26.1 - igor.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-igor:1.14.0 - kayenta.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-kayenta:0.19.0 - orca.yml: - artifactId: quay.io/opsmxpublic/ubi8-oes-orca:2.18.1 - rosco.yml: - artifactId: quay.io/opsmxpublic/ubi8-spin-rosco:0.23.0 - ## Uncomment if you want to add extra commands to the init script - ## run by the init container before halyard is started. - ## The content will be passed through `tpl`, so value interpolation is supported. - # additionalInitScript: |- - - ## Uncomment if you want to add annotations on halyard and install-using-hal pods - # annotations: - # iam.amazonaws.com/role: - - ## Uncomment the following resources definitions to control the cpu and memory - # resources allocated for the halyard pod - resources: - {} - # requests: - # memory: "1Gi" - # cpu: "100m" - # limits: - # memory: "2Gi" - # cpu: "200m" - - ## Uncomment if you want to set environment variables on the Halyard pod. - # env: - # - name: JAVA_OPTS - # value: -Dhttp.proxyHost=proxy.example.com - customCerts: - ## Enable to override the default cacerts with your own one - enabled: false - secretName: custom-cacerts - - # Define which registries and repositories you want available in your - # Spinnaker pipeline definitions - # For more info visit: - # https://www.spinnaker.io/setup/providers/docker-registry/ - - # Configure your Docker registries here - dockerRegistries: - - name: dockerhub - address: index.docker.io - repositories: - - library/alpine - - library/ubuntu - - library/centos - - library/nginx - # - name: gcr - # address: https://gcr.io - # username: _json_key - # password: '' - # email: 1234@5678.com - - # If you don't want to put your passwords into a values file - # you can use a pre-created secret instead of putting passwords - # (specify secret name in below `dockerRegistryAccountSecret`) - # per account above with data in the format: - # : - - # dockerRegistryAccountSecret: myregistry-secrets - - kubeConfig: - # Use this when you want to register arbitrary clusters with Spinnaker - # Upload your ~/kube/.config to a secret - enabled: false - secretName: my-kubeconfig - secretKey: config - # List of contexts from the kubeconfig to make available to Spinnaker - contexts: - - default - deploymentContext: default - omittedNameSpaces: - - kube-system - - kube-public - onlySpinnakerManaged: - enabled: true - - # spinnakerFeatureFlags is a list of Spinnaker feature flags to enable - # Ref: https://www.spinnaker.io/reference/halyard/commands/#hal-config-features-edit - # spinnakerFeatureFlags: - # - artifacts - # - pipeline-templates - spinnakerFeatureFlags: [] - - # Node labels for pod assignment - # Ref: https://kubernetes.io/docs/user-guide/node-selection/ - # nodeSelector to provide to each of the Spinnaker components - nodeSelector: {} - - # Redis password to use for the in-cluster redis service - # Enable redis to use in-cluster redis - redis: - enabled: true - # External Redis option will be enabled if in-cluster redis is disabled - external: - host: "" - port: 6379 - # password: "" - password: password - nodeSelector: {} - cluster: - enabled: false - - ## Redis config file - ## ref: https://redis.io/topics/config - ## - configmap: |- - # Enable AOF https://redis.io/topics/persistence#append-only-file - appendonly no - # Disable RDB persistence, AOF persistence already enabled. - save 60 1000 - - # Use ubi8 custom images - #image: - # registry: docker.io - # repository: devopsmx/ubi8-oes-redis - # tag: 4.0.14 - # Uncomment if you don't want to create a PVC for redis - # master: - # persistence: - # enabled: false - - # Minio access/secret keys for the in-cluster S3 usage - # Minio is not exposed publically - minio: - enabled: true - imageTag: RELEASE.2019-02-13T19-48-27Z - serviceType: ClusterIP - accessKey: spinnakeradmin - secretKey: spinnakeradmin - bucket: "spinnaker" - nodeSelector: {} - persistence: - enabled: true - size: 10Gi - # Use ubi8 custom images - #image: - # repository: devopsmx/ubi8-oes-minio - # tag: RELEASE.2019-09-18T21-55-05Z - # pullPolicy: IfNotPresent - - # Google Cloud Storage - gcs: - enabled: false - project: my-project-name - bucket: "" - ## if jsonKey is set, will create a secret containing it - jsonKey: "" - ## override the name of the secret to use for jsonKey, if `jsonKey` - ## is empty, it will not create a secret assuming you are creating one - ## external to the chart. the key for that secret should be `key.json`. - secretName: - - # AWS Simple Storage Service - s3: - enabled: false - bucket: "" - # rootFolder: "front50" - # region: "us-east-1" - # endpoint: "" - # accessKey: "" - # secretKey: "" - - # Azure Storage Account - azs: - enabled: false - # storageAccountName: "" - # accessKey: "" - # containerName: "spinnaker" - - rbac: - # Specifies whether RBAC resources should be created - create: true - - serviceAccount: - # Specifies whether a ServiceAccount should be created - create: true - # The name of the ServiceAccounts to use. - # If left blank it is auto-generated from the fullname of the release - halyardName: - spinnakerName: - securityContext: - # Specifies permissions to write for user/group - runAsUser: 1000 - fsGroup: 1000 - - ## Option to enable HA in Spinnaker; Enabled by default - enableHA: true - - ## Centralized Monitoring enable for Prometheus - enableCentralMonitoring: false - - # Initialize gitOps style Halyard - gitopsHalyard: - enabled: true - repo: - type: git # git, S3, stash(bitbucket server) use a different sample values file for s3/bitbuck-stash - ## Configure below fields only if repo type is git/stash. Skip to s3 section if type is s3 - baseUrlHostName: github.com # Specify it if git url is something other than github.com - organization: ksrinimba - # Specify project name only if repo is under a project - projectName: "" - repository: ryzon7-gitops - dynamicAccRepository: ryzon7-gitops - halConfigPath: / # Any other value is currently not supported - username: ksrinimba - token: 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 - secretName: opsmx-gitops-auth - # Promote applications and pipelines from one environment to another or take backup - pipelinePromotion: # GitHub only, Not supportd on S3 or Stash - enabled: true - type: github - organization: ksrinimba # Also called "project" in some repos - repository: ryzon7-gitops # bucket name in case of S3 - rootFolder: pipeline-promotion/ - ##### ONLY In case of S3 - AWS_ACCESS_KEY_ID: DFGHJ - AWS_SECRET_ACCESS_KEY: 45678kk - ##### S3 config for pipelinePromotion is complete - ##### For non-S3 repos - baseUrl: "github.com" # "git_url" - username: ksrinimba - token: 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 - branch: "kishore" - usermail: ksrinimba@gmail.com - #password="Krrish#438" ### Token is preferred, Password also might work, try your luck - # Instead of username, token/password, sshkey can be provided - #secret_sshkey="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC83mRU9Euwx/h547gNjMsWyA86MybQGRx6p9OyNp98RxLXjrG2B5D79F73Cp6eoA2I/zw4dxtPeIll88Yd0z4m0rp8p8FSmCrxOPauIzhPj5e4yRcEfLJ8LA3Q7Xl2Vfg1rUQ3PdaCpr4OZvy8i3kjGumoZ0rVvGW0LHMrDGXXz3FGeLD5dT/lpHJo1kbelQ/28K4xqdZQOOi0Z2oQemyiKdVifZBQR2NFaD72fQD5t0FbJLtgfm3Ko3sSFa00nITIfyUxCCRQbmd+Y+Dnl1dY9FS31sXXvxzJr62cn911igMbHBXvX77vQHoX7Bc9a0rrfnsulyzb5d11Iw0ywHcx kishore@kishore-Latitude-3510" - #API - #api_url="https://api.bitbucket.org/2.0/repositories/" # bitbucket - apiUrl: "https://api.github.com/repos" # guthub - #api_url="https://api.gitlab.com/repos" # gitLAB - #api_url="https://bb.opsmx.com/api" # bitbucket-server(stash) - - createPR: true - autoApprovePR: true - targetBranch: master - approvingUser: "ksrinimba" ### user who is going to auto-merge - approverToken: "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" ## Token for the user to auto-merge - #approver_password: "" ### Token is preferred, Password also might work, try it - #approver_secret_sshkey= ??? - ## x509 authentication for Spinnaker Gate - gatex509: - enabled: false - host: spingate-x509.domain.com - # Max time(in secs) that an init container of halyard should wait - # to fetch External Load Balancer IP of spin-deck and spin-gate - spinnakerLBCheckDelay: 1 # This is only used if not using ingress - mTLS: - enabled: false # Enable mTLS for Spinnaker Services and SSL for Deck and Gate - - ## Auth mechanism and credentials to be used by spin-cli for creating the sample pipelines - ## Here basic ldap auth is used by default; everything under spinCli will be pasted in ~/.spin/config - spinCli: - gate: - endpoint: http://spin-gate:8084 # Should same as the spin-gate URL - auth: - enabled: true - basic: - username: admin # Please updated as required - password: opsmxadmin123 # Please updated as required From 28fa8790031e97528ce8a3996f964b059945e77a Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 21:33:26 +0530 Subject: [PATCH 06/16] edit sample --- charts/oes/values.yaml | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/charts/oes/values.yaml b/charts/oes/values.yaml index bbcba6ad..bddd7881 100644 --- a/charts/oes/values.yaml +++ b/charts/oes/values.yaml @@ -1045,16 +1045,36 @@ spinnaker: # Promote applications and pipelines from one environment to another or take backup pipelinePromotion: # GitHub only, Not supportd on S3 or Stash enabled: true - organization: opsmx - repository: - usermail: - token: - baseURL: https://bbdev.opsmx.com - projectName: ksrin - dynamicAccRepository: test38 - executionNamespace: default # Do not change - executionAccountName: default # Do not change + type: github + organization: ksrinimba # Also called "project" in some repos + repository: ryzon7-gitops # bucket name in case of S3 + rootFolder: pipeline-promotion/ + ##### ONLY In case of S3 + AWS_ACCESS_KEY_ID: DFGHJ + AWS_SECRET_ACCESS_KEY: 45678kk + ##### S3 config for pipelinePromotion is complete + ##### For non-S3 repos + baseUrl: "github.com" # "git_url" + username: ksrinimba + token: ghp_38lL4E09LHxod331iZvS3VIGIF + branch: master + usermail: ksrinimba@gmail.com + #password="K438" ### Token is preferred, Password also might work, try your luck + # Instead of username, token/password, sshkey can be provided + #API + #api_url="https://api.bitbucket.org/2.0/repositories/" # bitbucket + apiUrl: "https://api.github.com/repos" # guthub + #api_url="https://api.gitlab.com/repos" # gitLAB + #api_url="https://bb.opsmx.com/api" # bitbucket-server(stash) + + createPR: false + autoApprovePR: false + targetBranch: master + approvingUser: "ksrinimba" ### user who is going to auto-merge + approverToken: "9MJn8LNva43EM8Js3b9opM" ## Token for the user to auto-merge + #approver_password: "" ### Token is preferred, Password also might work, try it + #approver_secret_sshkey= ??? + ## x509 authentication for Spinnaker Gate gatex509: enabled: false From 01663813f562c0891537887dc14a0639df2ae72f Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 21:39:20 +0530 Subject: [PATCH 07/16] Give permissions to all for openshift --- .../spinnaker/templates/configmap/halyard-init-script.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml index 605b5488..69a02a94 100755 --- a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml +++ b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml @@ -73,6 +73,7 @@ data: if [ -f /tmp/spinnaker/.hal/halyard.yaml ]; then cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config fi + chmod 777 -R /tmp/spinnaker {{- end }} # git or stash {{- end }} # Enabled @@ -112,7 +113,10 @@ data: cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config fi {{- end }} # End of S3 + chmod 777 -R /tmp/spinnaker {{- if .Values.gitopsHalyard.mTLS.enabled }} sed -i s/SPINNAKER_NAMESPACE/${SPINNAKER_NAMESPACE}/g /tmp/spinnaker/.hal/default/service-settings/* {{- end }} + chmod 777 -R /tmp/spinnaker + From b8fbecdc10370e3ba9a559623068e3883a27a129 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 21:52:41 +0530 Subject: [PATCH 08/16] more permissions --- .../spinnaker/templates/configmap/halyard-init-script.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml index 69a02a94..db10080f 100755 --- a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml +++ b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml @@ -73,7 +73,7 @@ data: if [ -f /tmp/spinnaker/.hal/halyard.yaml ]; then cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config fi - chmod 777 -R /tmp/spinnaker + chmod 777 -R /tmp/spinnaker/.hal {{- end }} # git or stash {{- end }} # Enabled @@ -113,10 +113,9 @@ data: cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config fi {{- end }} # End of S3 - chmod 777 -R /tmp/spinnaker {{- if .Values.gitopsHalyard.mTLS.enabled }} sed -i s/SPINNAKER_NAMESPACE/${SPINNAKER_NAMESPACE}/g /tmp/spinnaker/.hal/default/service-settings/* {{- end }} - chmod 777 -R /tmp/spinnaker + chmod 777 -R /tmp/spinnaker/.hal From 0527c57a6c341ebe24d4b8f2f2704aab54bcd4d1 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Wed, 30 Jun 2021 21:59:13 +0530 Subject: [PATCH 09/16] bug fix --- charts/oes/templates/pipeline-promotion/git-token-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml index 14f883f1..538f9a4a 100644 --- a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml @@ -3,7 +3,7 @@ apiVersion: v1 stringData: # Git token to access repo where pipeline stuff is stored git_secret_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.token }} - git_pr_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approver_token }} + git_pr_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approverToken }} #git_secret_sshkey: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} kind: Secret metadata: From f17401580c4142de1658a901ddbd3ad9d65f87d3 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Thu, 1 Jul 2021 16:38:07 +0530 Subject: [PATCH 10/16] Made changes as per Abhis comments --- .../configmap/halyard-init-script.yaml | 2 -- .../pipeline-promotion/git-token-secret.yaml | 1 - .../local-spin-cli-config-secret.yaml | 1 - .../pipe-promot-scripts-cm.yaml | 2 -- .../spin-cli-config-secret.yaml | 1 - charts/oes/values.yaml | 22 +++++++++---------- 6 files changed, 10 insertions(+), 19 deletions(-) diff --git a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml index db10080f..db262883 100755 --- a/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml +++ b/charts/oes/charts/spinnaker/templates/configmap/halyard-init-script.yaml @@ -73,7 +73,6 @@ data: if [ -f /tmp/spinnaker/.hal/halyard.yaml ]; then cp /tmp/spinnaker/.hal/halyard.yaml /tmp/config fi - chmod 777 -R /tmp/spinnaker/.hal {{- end }} # git or stash {{- end }} # Enabled @@ -117,5 +116,4 @@ data: {{- if .Values.gitopsHalyard.mTLS.enabled }} sed -i s/SPINNAKER_NAMESPACE/${SPINNAKER_NAMESPACE}/g /tmp/spinnaker/.hal/default/service-settings/* {{- end }} - chmod 777 -R /tmp/spinnaker/.hal diff --git a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml index 538f9a4a..7f52476a 100644 --- a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml @@ -7,7 +7,6 @@ stringData: #git_secret_sshkey: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} kind: Secret metadata: - #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} name: git-token type: Opaque {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml index d4fe9da1..d530590c 100644 --- a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml @@ -9,5 +9,4 @@ stringData: kind: Secret metadata: name: local-spin-cli-config - #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml index 0dfa2032..4d2c5831 100644 --- a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml +++ b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml @@ -569,6 +569,4 @@ data: \ fi\n fi\n fi\n}\n" kind: ConfigMap metadata: - creationTimestamp: null name: pipe-promot-scripts - namespace: pipe-test diff --git a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml index 031702df..d94aedeb 100644 --- a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml @@ -10,5 +10,4 @@ stringData: kind: Secret metadata: name: spin-cli-config - #namespace: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.executionNamespace }} {{- end }} diff --git a/charts/oes/values.yaml b/charts/oes/values.yaml index bddd7881..9a2d5c5e 100644 --- a/charts/oes/values.yaml +++ b/charts/oes/values.yaml @@ -1026,7 +1026,7 @@ spinnaker: gitopsHalyard: enabled: true repo: - type: git # git, S3, stash(bitbucket server) use a different sample values file for s3/bitbuck-stash + type: git # git, s3, stash(bitbucket server) use a different sample values file for s3/bitbuck-stash ## Configure below fields only if repo type is git/stash. Skip to s3 section if type is s3 baseUrlHostName: github.com # Specify it if git url is something other than github.com organization: OpsMx @@ -1045,9 +1045,9 @@ spinnaker: # Promote applications and pipelines from one environment to another or take backup pipelinePromotion: # GitHub only, Not supportd on S3 or Stash enabled: true - type: github - organization: ksrinimba # Also called "project" in some repos - repository: ryzon7-gitops # bucket name in case of S3 + type: git # git, s3, stash + organization: krish # Also called "project" in some repos + repository: my-gitops # bucket name in case of S3 rootFolder: pipeline-promotion/ ##### ONLY In case of S3 AWS_ACCESS_KEY_ID: DFGHJ @@ -1055,25 +1055,23 @@ spinnaker: ##### S3 config for pipelinePromotion is complete ##### For non-S3 repos baseUrl: "github.com" # "git_url" - username: ksrinimba - token: ghp_38lL4E09LHxod331iZvS3VIGIF + username: krish + token: lL4E09LHxod331iZvS3VIGIF branch: master - usermail: ksrinimba@gmail.com + usermail: krish@company.com #password="K438" ### Token is preferred, Password also might work, try your luck # Instead of username, token/password, sshkey can be provided #API #api_url="https://api.bitbucket.org/2.0/repositories/" # bitbucket apiUrl: "https://api.github.com/repos" # guthub #api_url="https://api.gitlab.com/repos" # gitLAB - #api_url="https://bb.opsmx.com/api" # bitbucket-server(stash) + #api_url="https://bbq.opsmx.com/api" # bitbucket-server(stash) createPR: false autoApprovePR: false targetBranch: master - approvingUser: "ksrinimba" ### user who is going to auto-merge - approverToken: "9MJn8LNva43EM8Js3b9opM" ## Token for the user to auto-merge - #approver_password: "" ### Token is preferred, Password also might work, try it - #approver_secret_sshkey= ??? + approvingUser: "krish" ### user who is going to auto-merge + approverToken: "9MJnNva43EM8Js3b9opM" ## Token for the user to auto-merge ## x509 authentication for Spinnaker Gate gatex509: From 76f67f07eb497fb0dce9abc12ec5777dd8af3bf6 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Thu, 1 Jul 2021 16:43:38 +0530 Subject: [PATCH 11/16] Fixed issue related to git_pr_token being empty --- charts/oes/templates/pipeline-promotion/git-token-secret.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml index 7f52476a..e062a04f 100644 --- a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml @@ -3,8 +3,9 @@ apiVersion: v1 stringData: # Git token to access repo where pipeline stuff is stored git_secret_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.token }} +{{- if .Values.spinnaker.gitopsHalyard.pipelinePromotion.approverToken }} git_pr_token: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.approverToken }} - #git_secret_sshkey: {{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} +{{- end }} kind: Secret metadata: name: git-token From cba230b91ece9d0a8688133e5c84da18701b6cc5 Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Thu, 1 Jul 2021 16:45:18 +0530 Subject: [PATCH 12/16] Bumped-up the chart number --- charts/oes/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/oes/Chart.yaml b/charts/oes/Chart.yaml index 9b206ded..3c1dc8b5 100644 --- a/charts/oes/Chart.yaml +++ b/charts/oes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: oes -version: 3.8.4 +version: 3.8.5 appVersion: 3.8.1 description: OES is a non-forked version of OSS spinnaker icon: https://raw.githubusercontent.com/OpsMx/enterprise-spinnaker/master/img/opsmx.png From 542399f4cf7bf557fb4ce6d89aa74efa383eb98b Mon Sep 17 00:00:00 2001 From: ksrinimba Date: Sun, 11 Jul 2021 16:57:23 +0530 Subject: [PATCH 13/16] Changes for Pipeline-promot in non-gitops --- .../charts/spinnaker/config/orca-local.yml | 146 ++++-------------- .../pipeline-promotion/git-token-secret.yaml | 3 + .../local-spin-cli-config-secret.yaml | 4 + .../pipe-promot-config-cm.yaml | 5 +- .../pipe-promot-scripts-cm.yaml | 4 + .../spin-cli-config-secret.yaml | 3 + 6 files changed, 52 insertions(+), 113 deletions(-) diff --git a/charts/oes/charts/spinnaker/config/orca-local.yml b/charts/oes/charts/spinnaker/config/orca-local.yml index 27fb572d..f3e33db6 100644 --- a/charts/oes/charts/spinnaker/config/orca-local.yml +++ b/charts/oes/charts/spinnaker/config/orca-local.yml @@ -1,3 +1,13 @@ +pollers: + oldPipelineCleanup: + enabled: true # This enables old pipeline execution cleanup (default: false) + intervalMs: 3600000 # How many milliseconds between pipeline cleanup runs (default: 1hr or 3600000) + thresholdDays: 30 # How old a pipeline execution must be to be deleted (default: 30) + minimumPipelineExecutions: 5 # How many executions to keep around (default: 5) + +tasks: + daysOfExecutionHistory: 180 # How many days to keep old task executions around. + job: preconfigured: kubernetes: @@ -38,6 +48,8 @@ job: volumeMounts: - mountPath: /home/opsmx/scripts name: pipe-promot-scripts + - mountPath: /home/opsmx/config + name: pipe-promot-config - mountPath: /home/opsmx/.spin name: spin-cli-config env: @@ -47,67 +59,21 @@ job: value: 'will be replaced' - name: command value: 'upload' - - name: git_friendly_username - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_friendly_username - - name: git_friendly_username - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_friendly_username - - name: git_project - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_project - - name: git_refresh_enabled - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_refresh_enabled - - name: git_repo - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_repo - - name: git_user_email - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_user_email - - name: ignore_spin_errors - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: ignore_spin_errors - name: git_secret_token valueFrom: secretKeyRef: name: git-token key: git_secret_token - - name: debug + - name: git_pr_token valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: debug - - name: pipelineconfig - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelineconfig - - name: pipelineconfigdir - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelineconfigdir - - name: pipelinecreateconf - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelinecreateconf + secretKeyRef: + name: git-token + key: git_pr_token volumes: + - configMap: + defaultMode: 420 + name: pipe-promot-config + name: pipe-promot-config - configMap: defaultMode: 420 name: pipe-promot-scripts @@ -155,6 +121,8 @@ job: volumeMounts: - mountPath: /home/opsmx/scripts name: pipe-promot-scripts + - mountPath: /home/opsmx/config + name: pipe-promot-config - mountPath: /home/opsmx/.spin name: spin-cli-config env: @@ -164,67 +132,21 @@ job: value: 'will be replaced' - name: command value: 'download' - - name: git_friendly_username - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_friendly_username - - name: git_friendly_username - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_friendly_username - - name: git_project - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_project - - name: git_refresh_enabled - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_refresh_enabled - - name: git_repo - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_repo - - name: git_user_email - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: git_user_email - - name: ignore_spin_errors - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: ignore_spin_errors - name: git_secret_token valueFrom: secretKeyRef: name: git-token key: git_secret_token - - name: debug + - name: git_pr_token valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: debug - - name: pipelineconfig - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelineconfig - - name: pipelineconfigdir - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelineconfigdir - - name: pipelinecreateconf - valueFrom: - configMapKeyRef: - name: pipe-promot-config - key: pipelinecreateconf + secretKeyRef: + name: git-token + key: git_pr_token volumes: + - configMap: + defaultMode: 420 + name: pipe-promot-config + name: pipe-promot-config - configMap: defaultMode: 420 name: pipe-promot-scripts @@ -241,7 +163,7 @@ job: cloudProvider: kubernetes account: default waitForCompletion: true - application: spin + application: sampleapp parameters: - defaultValue: "http://oes-sapor:8085" description: Please enter the POLICY Proxy Hostname and Port @@ -308,7 +230,7 @@ job: restartPolicy: Never - label: VerificationGate cloudProvider: kubernetes - application: spin + application: sampleapp description: Stage for Verification Gate propertyFile: verificationgate account: default @@ -414,7 +336,7 @@ job: - label: VisibilityApproval cloudProvider: kubernetes credentials: default - application: spin + application: sampleapp description: Stage for Visibility Approval propertyFile: visibilityapprove account: default @@ -548,7 +470,7 @@ job: restartPolicy: Never - label: TestVerification cloudProvider: kubernetes - application: spin + application: sampleapp description: Stage for TestVerification propertyFile: testverification account: default diff --git a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml index e062a04f..42b1e7a8 100644 --- a/charts/oes/templates/pipeline-promotion/git-token-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/git-token-secret.yaml @@ -9,5 +9,8 @@ stringData: kind: Secret metadata: name: git-token +{{- if not .Values.spinnaker.gitopsHalyard.enabled }} + namespace: default +{{- end }} type: Opaque {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml index d530590c..1de87dc5 100644 --- a/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/local-spin-cli-config-secret.yaml @@ -9,4 +9,8 @@ stringData: kind: Secret metadata: name: local-spin-cli-config +{{- if not .Values.spinnaker.gitopsHalyard.enabled }} + namespace: default +{{- end }} + {{- end }} diff --git a/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml b/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml index 124c3e02..0f12b979 100644 --- a/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml +++ b/charts/oes/templates/pipeline-promotion/pipe-promot-config-cm.yaml @@ -36,5 +36,8 @@ data: #git_secret_sshkey={{ .Values.spinnaker.gitopsHalyard.pipelinePromotion.secret_sshkey }} kind: ConfigMap metadata: - creationTimestamp: null name: pipe-promot-config +{{- if not .Values.spinnaker.gitopsHalyard.enabled }} + namespace: default +{{- end }} + diff --git a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml index 4d2c5831..d1b3cf1e 100644 --- a/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml +++ b/charts/oes/templates/pipeline-promotion/pipe-promot-scripts-cm.yaml @@ -570,3 +570,7 @@ data: kind: ConfigMap metadata: name: pipe-promot-scripts +{{- if not .Values.spinnaker.gitopsHalyard.enabled }} + namespace: default +{{- end }} + diff --git a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml index d94aedeb..43a0401e 100644 --- a/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml +++ b/charts/oes/templates/pipeline-promotion/spin-cli-config-secret.yaml @@ -10,4 +10,7 @@ stringData: kind: Secret metadata: name: spin-cli-config +{{- if not .Values.spinnaker.gitopsHalyard.enabled }} + namespace: default +{{- end }} {{- end }} From 3a8c82edbda79fa43bdd2dce4c6fec58e63bb711 Mon Sep 17 00:00:00 2001 From: Pranav-b-7 Date: Thu, 29 Jul 2021 14:02:08 +0530 Subject: [PATCH 14/16] migration script from v3.8.x to v3.9 --- .../migration_v3.8.x_to_v3.9.py | 324 ++++++++++++++++++ .../steps_to_migrate_from_v3.8.x_to_v3.9.txt | 10 + 2 files changed, 334 insertions(+) create mode 100644 scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py create mode 100644 scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt diff --git a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py new file mode 100644 index 00000000..97423df6 --- /dev/null +++ b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py @@ -0,0 +1,324 @@ +import json +import psycopg2 +import sys +import datetime +import string +import random +import requests + +def perform_migration(input_string): + try: + print ('Migrating from v3.8.x to v3.9') + + # Modify existing user_group table + modifyUserGroupTable() + + # add admin groups configuration to table + print ('Printing all admin group names given:: ', input_string) + admin_groups = input_string.split(',') + for group_name in admin_groups: + map_user_group_admin(group_name) + print("Successfully configured admin groups to table to v3.9") + + print("Migrating approval gate parameter entity") + alter_approval_gate_parameter() + approval_gate_instance_ids = fetch_distinct_gate_instances() + for approval_gate_instance_id in approval_gate_instance_ids: + connectors = fetch_distinct_connectors(approval_gate_instance_id[0]) + param_group_id = 1 + for connector in connectors: + update_param_group_id(param_group_id, connector[0], approval_gate_instance_id[0]) + param_group_id = param_group_id + 1 + + print("Migrating pipeline json") + pipeline_jsons = fetch_pipeline_json() + for pipeline_json in pipeline_jsons: + migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) + print("Successfully migrated to v3.9") + + platform_conn.commit() + visibility_conn.commit() + except Exception as e: + platform_conn.rollback() + visibility_conn.rollback() + print ('Exception occured during migration : ', e) + finally: + platform_conn.close() + visibility_conn.close() + + +def alter_approval_gate_parameter(): + try: + cur = visibility_conn.cursor() + cur.execute("ALTER TABLE approval_gate_parameter ADD COLUMN IF NOT EXISTS param_group_id INT NOT NULL DEFAULT 0") + except Exception as e: + print("Exception occured while altering the approval_gate_parameter table : ", e) + raise e + + +def fetch_distinct_gate_instances(): + try: + cur = visibility_conn.cursor() + cur.execute("select distinct(approval_gate_instance_id) from approval_gate_parameter ORDER BY approval_gate_instance_id") + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching distinct gate instances : ", e) + raise e + + +def fetch_distinct_connectors(gate_instance_id): + try: + cur = visibility_conn.cursor() + cur.execute("select distinct(connector_type) from approval_gate_parameter where approval_gate_instance_id = "+str(gate_instance_id)) + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching distinct connectors : ", e) + raise e + + +def update_param_group_id(param_group_id, connector_type, approval_gate_instance_id): + try: + cur = visibility_conn.cursor() + cur.execute("update approval_gate_parameter set param_group_id = " + str(param_group_id) + " where connector_type = '" + str(connector_type) + "' AND approval_gate_instance_id = "+str(approval_gate_instance_id)) + except Exception as e: + print("Exception occured while updating param group id : ", e) + raise e + + +def fetch_pipeline_json(): + try: + cur = platform_conn.cursor() + cur.execute("select id, pipeline_json from pipeline") + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching pipeline jsons : ", e) + raise e + + +def migrate_pipeline_json(id, pipeline_json): + try: + cur = platform_conn.cursor() + pipe_json = json.loads(pipeline_json) + stages = pipe_json['stages'] + if stages is not None and len(stages)>0: + for stage in stages: + gate_type = stage['type'] + if gate_type is not None and len(gate_type)>0: + if str(gate_type).strip() == 'Visibility Approval': + migrate_approval_gate(stage) + elif str(gate_type).strip() == 'Verification': + migrate_verification_gate(stage) + elif str(gate_type).strip() == 'Test Verification': + migrate_test_verification_gate(stage) + elif str(gate_type).strip() == 'Policy Stage': + migrate_policy_gate(stage) + pipe_json['stages'] = stages + cur.execute("update pipeline set pipeline_json = '"+str(json.dumps(pipe_json))+"' where id="+str(id)) + + except Exception as e: + print("Exception occured while migrating pipeline json : ", e) + raise e + + +def migrate_policy_gate(stage): + del stage['alias'] + stage['type'] = 'policy' + + +def migrate_test_verification_gate(stage): + del stage['alias'] + del stage['comments'] + stage['type'] = 'testVerification' + + +def migrate_verification_gate(stage): + del stage['alias'] + del stage['comments'] + stage['type'] = 'verification' + + +def migrate_approval_gate(stage): + try: + del stage['alias'] + stage['type'] = 'approval' + parameters = stage['parameters'] + if parameters is not None and len(parameters) > 0: + canaryid = parameters['canaryid'] + gitrepo = parameters['gitrepo'] + jenkinsartifact = parameters['jenkinsartifact'] + jenkinsbuild = parameters['jenkinsbuild'] + appscanid = parameters['appscanid'] + aquawave = parameters['aquawave'] + jenkinsjob = parameters['jenkinsjob'] + projectkey = parameters['projectkey'] + connector = parameters['connector'] + jiraid = parameters['jiraid'] + header = parameters['header'] + gitcommitid = parameters['gitcommitid'] + connectordata = parameters['connectordata'] + imageids = parameters['imageids'] + gateurl = parameters['gateurl'] + + connectors = [] + if gateurl is not None and len(gateurl)>0: + url_components = str(gateurl).split("/") + approval_gate_id = url_components[len(url_components) - 2] + configured_tool_connectors = fetch_configured_tool_connectors(approval_gate_id) + for configured_tool_connector in configured_tool_connectors: + connector_type = configured_tool_connector['connectorType'] + if connector_type is not None: + if str(connector_type).strip().upper() == 'JIRA': + configure_jira(configured_tool_connector, jiraid) + elif str(connector_type).strip().upper() == 'GIT': + configure_git(configured_tool_connector, gitcommitid, gitrepo) + elif str(connector_type).strip().upper() == 'JENKINS': + configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob) + elif str(connector_type).strip().upper() == 'AUTOPILOT': + configure_autopilot(canaryid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'SONARQUBE': + configure_sonarqube(configured_tool_connector, projectkey) + elif str(connector_type).strip().upper() == 'APPSCAN': + configure_appscan(appscanid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'AQUAWAVE': + configure_aquawave(aquawave, configured_tool_connector) + connectors.append(configured_tool_connector) + gateurl = str(gateurl).strip().replace("v4", "v5") + gateurl = str(gateurl).strip().replace("v3", "v5") + gateurl = str(gateurl).strip().replace("v2", "v5") + gateurl = str(gateurl).strip().replace("v1", "v5") + new_parameters = {"imageIds": imageids, + "connectors": connectors, + "gateUrl": gateurl} + stage['parameters'] = new_parameters + + except Exception as e: + print("Exception occured while migrating approval gate : ", e) + raise e + + +def configure_aquawave(aquawave, configured_tool_connector): + aqua = [] + param = {"imageId": aquawave} + aqua.append(param) + configured_tool_connector['values'] = aqua + + +def configure_appscan(appscanid, configured_tool_connector): + appscan = [] + param = {"id": appscanid} + appscan.append(param) + configured_tool_connector['values'] = appscan + + +def configure_sonarqube(configured_tool_connector, projectkey): + sonarqube = [] + param = {"projectKey": projectkey} + sonarqube.append(param) + configured_tool_connector['values'] = sonarqube + + +def configure_autopilot(canaryid, configured_tool_connector): + autopilot = [] + param = {"canaryId": canaryid} + autopilot.append(param) + configured_tool_connector['values'] = autopilot + + +def configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob): + jenkins = [] + param = {"job": jenkinsjob, + "buildId": jenkinsbuild, + "artifact": jenkinsartifact} + jenkins.append(param) + configured_tool_connector['values'] = jenkins + + +def configure_git(configured_tool_connector, gitcommitid, gitrepo): + git = [] + param = {"repo": gitrepo, + "commitId": gitcommitid} + git.append(param) + configured_tool_connector['values'] = git + + +def configure_jira(configured_tool_connector, jiraid): + jira_tickets = [] + if jiraid is not None and len(jiraid) > 0: + jiraids = str(jiraid).split(",") + for id in jiraids: + param = {"jira_ticket_no": id} + jira_tickets.append(param) + configured_tool_connector['values'] = jira_tickets + + +def fetch_configured_tool_connectors(approval_gate_id): + try: + url = oes_visibility_url + "/visibilityservice/v1/approvalGates/"+str(approval_gate_id)+"/configuredtoolConnectors" + headers = {'x-spinnaker-user': oes_admin_user} + response = requests.get(url=url, headers=headers) + return json.loads(response.content) + except Exception as e: + print("Exception occured while fetching configured tool connectors : ", e) + raise e + +def modifyUserGroupTable(): + try: + cur = platform_conn.cursor() + cur.execute("""ALTER TABLE user_group ADD COLUMN IF NOT EXISTS "is_admin" BOOLEAN NOT NULL DEFAULT FALSE""" + ) + except Exception as e: + print ('Exception occured while modifying user_group table: ', + e) + raise e + + +def map_user_group_admin(group_name): + try: + cur = platform_conn.cursor() + cur.execute("SELECT id FROM user_group Where name='" + + group_name + "'") + user_group_id = cur.fetchone() + update_user_group(user_group_id) + except Exception as e: + print ('Exception occured while fetching data from user_group table: ' + , e) + raise e + + +def update_user_group(user_group_id): + try: + cur = platform_conn.cursor() + cur.execute('UPDATE user_group SET is_admin = true WHERE id = '+str(user_group_id[0])) + except Exception as e: + print ('Exception occured while updating user_group table: ', e) + raise e + +if __name__ == '__main__': + n = len(sys.argv) + if n != 8: + print ('Please pass valid 7 arguments ') + + platform_db = sys.argv[1] + platform_host = sys.argv[2] + visibility_db = sys.argv[3] + visibility_host = sys.argv[4] + port = sys.argv[5] + oes_visibility_url = sys.argv[6] + oes_admin_user = sys.argv[7] + + # Establishing the platform db connection + + platform_conn = psycopg2.connect(database=platform_db, + user='postgres', password='networks123', + host=platform_host, port=port) + print ('Opened platform database connection successfully') + + visibility_conn = psycopg2.connect(database=visibility_db, user='postgres', password='networks123', host=visibility_host, port=port) + print("Visibility database connection established successfully") + + # Getting input of admin groups + + input_string = \ + input('Enter admin groups to be configure in separated by comma:: ' + ) + perform_migration(input_string) diff --git a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt new file mode 100644 index 00000000..333c802e --- /dev/null +++ b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt @@ -0,0 +1,10 @@ +Steps to migrate from v3.8.x to v3.9 + +1. Copy the file : migration_v3.8.x_to_v3.9.py to some path in platform pod. +2. Run the below command to execute the script. + + python3 + + Replace with valid values + + Ex: python3 migration_v3.8.x_to_v3.9.py platformdb oes-db visibilitydb oes-db 5432 http://oes-visibility:8096 admin From c67425add40b9302c2d36b017d9ee38057cc20dd Mon Sep 17 00:00:00 2001 From: Sudhakar Guvvalapall Date: Fri, 30 Jul 2021 17:18:43 +0530 Subject: [PATCH 15/16] OP-7058: Added sapor db migration table alter for cloud configuration (#170) * OP-7058: Added sapor db migration table alter for cloud configuration * reverted with steps --- .../migration_v3.8.x_to_v3.9.py | 39 ++++++++++++++++--- .../steps_to_migrate_from_v3.8.x_to_v3.9.txt | 6 +-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py index 97423df6..52478aa1 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py +++ b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py @@ -34,18 +34,24 @@ def perform_migration(input_string): pipeline_jsons = fetch_pipeline_json() for pipeline_json in pipeline_jsons: migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) + + print("Migrating Cloud Configuration entity") + alter_cloud_configuration() + print("Successfully configured Cloud Configuration entity to table to v3.9") print("Successfully migrated to v3.9") platform_conn.commit() visibility_conn.commit() + oesdb_conn.commit() except Exception as e: platform_conn.rollback() visibility_conn.rollback() + oesdb_conn.rollback() print ('Exception occured during migration : ', e) finally: platform_conn.close() visibility_conn.close() - + oesdb_conn.close() def alter_approval_gate_parameter(): try: @@ -293,18 +299,30 @@ def update_user_group(user_group_id): print ('Exception occured while updating user_group table: ', e) raise e +def alter_cloud_configuration(): + try: + cur = oesdb_conn.cursor() + cur.execute("ALTER TABLE cloud_configuration ALTER COLUMN bakery_secret_key TYPE varchar(10000) USING bakery_secret_key::varchar") + cur.execute("ALTER TABLE cloud_configuration ALTER COLUMN secret_key TYPE varchar(10000) USING secret_key::varchar") + except Exception as e: + print("Exception occured while altering the cloud_configuration table : ", e) + raise e + + if __name__ == '__main__': n = len(sys.argv) - if n != 8: - print ('Please pass valid 7 arguments ') + if n != 10: + print ('Please pass valid 9 arguments ') platform_db = sys.argv[1] platform_host = sys.argv[2] visibility_db = sys.argv[3] visibility_host = sys.argv[4] - port = sys.argv[5] - oes_visibility_url = sys.argv[6] - oes_admin_user = sys.argv[7] + oes_db = sys.argv[5] + oes_host = sys.argv[6] + oes_visibility_url = sys.argv[7] + port = sys.argv[8] + oes_admin_user = sys.argv[9] # Establishing the platform db connection @@ -315,10 +333,19 @@ def update_user_group(user_group_id): visibility_conn = psycopg2.connect(database=visibility_db, user='postgres', password='networks123', host=visibility_host, port=port) print("Visibility database connection established successfully") + + + # Establishing the sapor db connection + oesdb_conn = psycopg2.connect(database=oes_db, + user='postgres', password='networks123', + host=oes_host, port=port) + print ('Opened sapor database connection successfully') + # Getting input of admin groups input_string = \ input('Enter admin groups to be configure in separated by comma:: ' ) perform_migration(input_string) + diff --git a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt index 333c802e..b31d439d 100644 --- a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt +++ b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt @@ -3,8 +3,8 @@ Steps to migrate from v3.8.x to v3.9 1. Copy the file : migration_v3.8.x_to_v3.9.py to some path in platform pod. 2. Run the below command to execute the script. - python3 + python3 - Replace with valid values + Replace with valid values - Ex: python3 migration_v3.8.x_to_v3.9.py platformdb oes-db visibilitydb oes-db 5432 http://oes-visibility:8096 admin + Ex: python3 migration_v3.8.x_to_v3.9.py platformdb oes-db visibilitydb oes-db oesdb oes-db http://oes-visibility:8096 5432 admin From 3f4588a185b05d2be9cc4e88f41bfc08fbb3dfbb Mon Sep 17 00:00:00 2001 From: Pranav-b-7 Date: Fri, 30 Jul 2021 17:33:14 +0530 Subject: [PATCH 16/16] Made some changes --- .../migration_v3.8.x_to_v3.9.py | 93 ++++++++++--------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py index 52478aa1..686d3579 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py +++ b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py @@ -33,7 +33,8 @@ def perform_migration(input_string): print("Migrating pipeline json") pipeline_jsons = fetch_pipeline_json() for pipeline_json in pipeline_jsons: - migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) + if pipeline_json is not None and len(pipeline_json) > 0: + migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) print("Migrating Cloud Configuration entity") alter_cloud_configuration() @@ -105,21 +106,22 @@ def migrate_pipeline_json(id, pipeline_json): try: cur = platform_conn.cursor() pipe_json = json.loads(pipeline_json) - stages = pipe_json['stages'] - if stages is not None and len(stages)>0: - for stage in stages: - gate_type = stage['type'] - if gate_type is not None and len(gate_type)>0: - if str(gate_type).strip() == 'Visibility Approval': - migrate_approval_gate(stage) - elif str(gate_type).strip() == 'Verification': - migrate_verification_gate(stage) - elif str(gate_type).strip() == 'Test Verification': - migrate_test_verification_gate(stage) - elif str(gate_type).strip() == 'Policy Stage': - migrate_policy_gate(stage) - pipe_json['stages'] = stages - cur.execute("update pipeline set pipeline_json = '"+str(json.dumps(pipe_json))+"' where id="+str(id)) + if len(pipe_json) > 0: + stages = pipe_json['stages'] + if stages is not None and len(stages)>0: + for stage in stages: + gate_type = stage['type'] + if gate_type is not None and len(gate_type)>0: + if str(gate_type).strip() == 'Visibility Approval': + migrate_approval_gate(stage) + elif str(gate_type).strip() == 'Verification': + migrate_verification_gate(stage) + elif str(gate_type).strip() == 'Test Verification': + migrate_test_verification_gate(stage) + elif str(gate_type).strip() == 'Policy Stage': + migrate_policy_gate(stage) + pipe_json['stages'] = stages + cur.execute("update pipeline set pipeline_json = '"+str(json.dumps(pipe_json))+"' where id="+str(id)) except Exception as e: print("Exception occured while migrating pipeline json : ", e) @@ -170,38 +172,45 @@ def migrate_approval_gate(stage): url_components = str(gateurl).split("/") approval_gate_id = url_components[len(url_components) - 2] configured_tool_connectors = fetch_configured_tool_connectors(approval_gate_id) - for configured_tool_connector in configured_tool_connectors: - connector_type = configured_tool_connector['connectorType'] - if connector_type is not None: - if str(connector_type).strip().upper() == 'JIRA': - configure_jira(configured_tool_connector, jiraid) - elif str(connector_type).strip().upper() == 'GIT': - configure_git(configured_tool_connector, gitcommitid, gitrepo) - elif str(connector_type).strip().upper() == 'JENKINS': - configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob) - elif str(connector_type).strip().upper() == 'AUTOPILOT': - configure_autopilot(canaryid, configured_tool_connector) - elif str(connector_type).strip().upper() == 'SONARQUBE': - configure_sonarqube(configured_tool_connector, projectkey) - elif str(connector_type).strip().upper() == 'APPSCAN': - configure_appscan(appscanid, configured_tool_connector) - elif str(connector_type).strip().upper() == 'AQUAWAVE': - configure_aquawave(aquawave, configured_tool_connector) - connectors.append(configured_tool_connector) - gateurl = str(gateurl).strip().replace("v4", "v5") - gateurl = str(gateurl).strip().replace("v3", "v5") - gateurl = str(gateurl).strip().replace("v2", "v5") - gateurl = str(gateurl).strip().replace("v1", "v5") - new_parameters = {"imageIds": imageids, - "connectors": connectors, - "gateUrl": gateurl} - stage['parameters'] = new_parameters + if configured_tool_connectors is not None and len(configured_tool_connectors) > 0: + for configured_tool_connector in configured_tool_connectors: + connector_type = configured_tool_connector['connectorType'] + if connector_type is not None: + if str(connector_type).strip().upper() == 'JIRA': + configure_jira(configured_tool_connector, jiraid) + elif str(connector_type).strip().upper() == 'GIT': + configure_git(configured_tool_connector, gitcommitid, gitrepo) + elif str(connector_type).strip().upper() == 'JENKINS': + configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob) + elif str(connector_type).strip().upper() == 'AUTOPILOT': + configure_autopilot(canaryid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'SONARQUBE': + configure_sonarqube(configured_tool_connector, projectkey) + elif str(connector_type).strip().upper() == 'APPSCAN': + configure_appscan(appscanid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'AQUAWAVE': + configure_aquawave(aquawave, configured_tool_connector) + connectors.append(configured_tool_connector) + set_new_parameters(connectors, gateurl, imageids, stage) + else: + set_new_parameters(connectors, gateurl, imageids, stage) except Exception as e: print("Exception occured while migrating approval gate : ", e) raise e +def set_new_parameters(connectors, gateurl, imageids, stage): + gateurl = str(gateurl).strip().replace("v4", "v5") + gateurl = str(gateurl).strip().replace("v3", "v5") + gateurl = str(gateurl).strip().replace("v2", "v5") + gateurl = str(gateurl).strip().replace("v1", "v5") + new_parameters = {"imageIds": imageids, + "connectors": connectors, + "gateUrl": gateurl} + stage['parameters'] = new_parameters + + def configure_aquawave(aquawave, configured_tool_connector): aqua = [] param = {"imageId": aquawave}