diff --git a/Release.txt b/Release.txt index 2a1d056..ce29be1 100644 --- a/Release.txt +++ b/Release.txt @@ -8,4 +8,9 @@ 3. Bug fixes ##### January 2024 Release ######## 1. External Test Credentials from post environment files -2. Project ID fix: Create a temp flow to export import project to maintain id in all environment. \ No newline at end of file +2. Project ID fix: Create a temp flow to export import project to maintain id in all environment. +##### July 2024 Release ######## +1. Support for API +2. Support for Parameters (with environment specific values) +3. Environemnt specific configs filenames aligned from environment config yaml. +4. Bug fixes \ No newline at end of file diff --git a/configs/dev.yml b/configs/dev.yml deleted file mode 100644 index 55ebea6..0000000 --- a/configs/dev.yml +++ /dev/null @@ -1,5 +0,0 @@ -tenant: - hostname: "env209597.int-az-eu.webmethods.io" # webMethodsIO Integration hostname or ip address - port: "443" # webMethodsIO Integration port - admin_username: "refplatform" # webMethodsIO Integration user - type: "dev" #DEV diff --git a/configs/env/dev.yml b/configs/env/dev.yml new file mode 100644 index 0000000..91abe6b --- /dev/null +++ b/configs/env/dev.yml @@ -0,0 +1,5 @@ +tenant: + hostname: "psprod.int-aws-de.webmethods.io" # webMethodsIO Integration hostname or ip address + port: "443" # webMethodsIO Integration port + admin_username: "gitops" # webMethodsIO Integration user + type: "dev" #DEV \ No newline at end of file diff --git a/configs/env/play.yml b/configs/env/play.yml new file mode 100644 index 0000000..2218d61 --- /dev/null +++ b/configs/env/play.yml @@ -0,0 +1,5 @@ +tenant: + hostname: "psdev.int-aws-de.webmethods.io" # webMthodsIO Integration hostname or ip address + port: "443" # webMthodsIO Integration port + admin_username: "gitops" # webMthodsIO Integration user + type: "play" #Dev \ No newline at end of file diff --git a/configs/prod.yml b/configs/env/prod.yml similarity index 100% rename from configs/prod.yml rename to configs/env/prod.yml diff --git a/configs/qa.yml b/configs/env/qa.yml similarity index 100% rename from configs/qa.yml rename to configs/env/qa.yml diff --git a/configs/play.yml b/configs/play.yml deleted file mode 100644 index b784726..0000000 --- a/configs/play.yml +++ /dev/null @@ -1,5 +0,0 @@ -tenant: - hostname: "insrsaaz.int-az-eu.webmethods.io" # webMethodsIO Integration hostname or ip address - port: "443" # webMethodsIO Integration port - admin_username: "refplatform" # webMthodsIO Integration user - type: "play" #Dev diff --git a/pipelines/InitiateTesting.yml b/pipelines/InitiateTesting.yml index 84647f3..b66fb74 100644 --- a/pipelines/InitiateTesting.yml +++ b/pipelines/InitiateTesting.yml @@ -50,7 +50,7 @@ steps: cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/${source}.yml`" + echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/${source}.yml`" echo $(System.DefaultWorkingDirectory) workingDirectory: '$(System.DefaultWorkingDirectory)' diff --git a/pipelines/initialize_pipeline.yml b/pipelines/initialize_pipeline.yml index 9bc9538..ae7e900 100644 --- a/pipelines/initialize_pipeline.yml +++ b/pipelines/initialize_pipeline.yml @@ -51,23 +51,23 @@ steps: cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/play.yml`" - echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/play.yml`" - echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/play.yml`" - echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/play.yml`" + echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env/play.yml`" + echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/play.yml`" + echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/play.yml`" + echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/play.yml`" echo "##vso[task.setvariable variable=repo_user]`yq -e ".repo.user" configs/repo.yml`" - echo "##vso[task.setvariable variable=dev_environment_hostname]`yq -e ".tenant.hostname" configs/dev.yml`" - echo "##vso[task.setvariable variable=dev_environment_port]`yq -e ".tenant.port" configs/dev.yml`" - echo "##vso[task.setvariable variable=dev_admin_user]`yq -e ".tenant.admin_username" configs/dev.yml`" + echo "##vso[task.setvariable variable=dev_environment_hostname]`yq -e ".tenant.hostname" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=dev_environment_port]`yq -e ".tenant.port" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=dev_admin_user]`yq -e ".tenant.admin_username" configs/env/dev.yml`" - echo "##vso[task.setvariable variable=qa_environment_hostname]`yq -e ".tenant.hostname" configs/qa.yml`" - echo "##vso[task.setvariable variable=qa_environment_port]`yq -e ".tenant.port" configs/qa.yml`" - echo "##vso[task.setvariable variable=qa_admin_user]`yq -e ".tenant.admin_username" configs/qa.yml`" + echo "##vso[task.setvariable variable=qa_environment_hostname]`yq -e ".tenant.hostname" configs/env/qa.yml`" + echo "##vso[task.setvariable variable=qa_environment_port]`yq -e ".tenant.port" configs/env/qa.yml`" + echo "##vso[task.setvariable variable=qa_admin_user]`yq -e ".tenant.admin_username" configs/env/qa.yml`" - echo "##vso[task.setvariable variable=prod_environment_hostname]`yq -e ".tenant.hostname" configs/prod.yml`" - echo "##vso[task.setvariable variable=prod_environment_port]`yq -e ".tenant.port" configs/prod.yml`" - echo "##vso[task.setvariable variable=prod_admin_user]`yq -e ".tenant.admin_username" configs/prod.yml`" + echo "##vso[task.setvariable variable=prod_environment_hostname]`yq -e ".tenant.hostname" configs/env/prod.yml`" + echo "##vso[task.setvariable variable=prod_environment_port]`yq -e ".tenant.port" configs/env/prod.yml`" + echo "##vso[task.setvariable variable=prod_admin_user]`yq -e ".tenant.admin_username" configs/env/prod.yml`" echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-$(repoName)" echo $(System.DefaultWorkingDirectory) diff --git a/pipelines/scripts/exportAsset.sh b/pipelines/scripts/exportAsset.sh index 2b58c6b..0dd745e 100644 --- a/pipelines/scripts/exportAsset.sh +++ b/pipelines/scripts/exportAsset.sh @@ -15,6 +15,7 @@ HOME_DIR=$7 synchProject=$8 source_type=$9 inlcudeAllReferenceData=${10} +envTypes=${11} debug=${@: -1} if [ -z "$LOCAL_DEV_URL" ]; then @@ -56,7 +57,14 @@ debug=${@: -1} echo "Missing template parameter source_type" exit 1 fi - + if [ -z "$inlcudeAllReferenceData" ]; then + echo "Missing template parameter inlcudeAllReferenceData" + exit 1 + fi + if [ -z "$envTypes" ]; then + echo "Missing template parameter envTypes" + exit 1 + fi if [ "$debug" == "debug" ]; then echo "******* Running in Debug mode ******" fi @@ -113,9 +121,7 @@ function exportSingleReferenceData () { metadataJson=$(echo "$metadataJson"| jq 'del(.columnNames, .dataRecords, .revisionData)') echo "$metadataJson" > metadata.json echo "$datajson" > ${source_type}.csv - cp -n ./${source_type}.csv dev.csv - cp -n ./${source_type}.csv qa.csv - cp -n ./${source_type}.csv prod.csv + configPerEnv . ${envTypes} "referenceData" ${source_type}.csv cd - fi cd ${HOME_DIR}/${repoName} @@ -173,6 +179,28 @@ function exportReferenceData (){ cd ${HOME_DIR}/${repoName} } +function configPerEnv(){ + localtion=$1 + envTypes=$2 + configType=$3 + sourceFile=$4 + key=$5 + + IFS=, read -ra values <<< "$envTypes" + for v in "${values[@]}" + do + # things with "$v" + if [ ${configType} == "referenceData" ]; then + cp ./$sourceFile ./$v.csv + else + if [ ${configType} == "project_parameter" ]; then + cp ./$sourceFile ./${key}_${v}.json + fi + fi + done + +} + function exportAsset(){ LOCAL_DEV_URL=$1 @@ -216,22 +244,28 @@ function exportAsset(){ echod "Rest_API Export:" ${EXPORT_URL} "with JSON: "${rest_api_json} echod $(ls -ltr) else - if [[ $assetType = workflow* ]]; then + if [[ $assetType = project_parameter* ]]; then echod $assetType - EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflows/${assetID}/export - cd ${HOME_DIR}/${repoName} - mkdir -p ./assets/workflows - cd ./assets/workflows - echod "Workflow Export:" ${EXPORT_URL} - echod $(ls -ltr) + exportProjectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} + return else - if [[ $assetType = flowservice* ]]; then - EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flows/${assetID}/export + if [[ $assetType = workflow* ]]; then + echod $assetType + EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflows/${assetID}/export cd ${HOME_DIR}/${repoName} - mkdir -p ./assets/flowservices - cd ./assets/flowservices - echo "Flowservice Export:" ${EXPORT_URL} + mkdir -p ./assets/workflows + cd ./assets/workflows + echod "Workflow Export:" ${EXPORT_URL} echod $(ls -ltr) + else + if [[ $assetType = flowservice* ]]; then + EXPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flows/${assetID}/export + cd ${HOME_DIR}/${repoName} + mkdir -p ./assets/flowservices + cd ./assets/flowservices + echo "Flowservice Export:" ${EXPORT_URL} + echod $(ls -ltr) + fi fi fi fi @@ -246,7 +280,6 @@ function exportAsset(){ --header 'Accept: application/json' \ -u ${admin_user}:${admin_password}) fi - downloadURL=$(echo "$linkJson" | jq -r '.output.download_link') regex='(https?|ftp|file)://[-[:alnum:]\+&@#/%?=~_|!:,.;]*[-[:alnum:]\+&@#/%=~_|]' @@ -278,6 +311,74 @@ function exportAsset(){ } +function exportProjectParameters(){ + + LOCAL_DEV_URL=$1 + admin_user=$2 + admin_password=$3 + repoName=$4 + assetID=$5 + assetType=$6 + HOME_DIR=$7 + synchProject=$8 + inlcudeAllReferenceData=$9 + + if [ ${synchProject} == true ]; then + PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params + else + PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${assetID} + fi + + ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + -u ${admin_user}:${admin_password}) + + ppListExport=$(echo "$ppListJson" | jq '. // empty') + + if [ -z "$ppListExport" ]; then + echo "No Project Parameters retreived:" ${ppListJson} + else + mkdir -p ./assets/projectConfigs/parameters + cd ./assets/projectConfigs/parameters + if [ ${synchProject} != true ]; then + parameterUID=$(jq -r '.output.uid' <<< "$ppListJson") + mkdir -p ./${parameterUID} + cd ./${parameterUID} + data=$(jq -r '.output.param' <<< "$ppListJson") + key=$(jq -r '.output.param.key' <<< "$ppListJson") + metadataJson='{ "uid":"'${parameterUID}'" }' + echo ${metadataJson} > ./metadata.json + echo ${data} > ./${key}_${source_type}.json + configPerEnv . ${envTypes} "project_parameter" ${key}_${source_type}.json ${key} + #cp -n ./${key}_${source_type}.json ${key}_dev.json + #cp -n ./${key}_${source_type}.json ${key}_qa.json + #cp -n ./${key}_${source_type}.json ${key}_prod.json + cd .. + else + for item in $(jq -c -r '.output[]' <<< "$ppListJson"); do + echod "Inside Parameters Loop" + parameterUID=$(jq -r '.uid' <<< "$item") + mkdir -p ./${parameterUID} + cd ./${parameterUID} + data=$(jq -r '.param' <<< "$item") + key=$(jq -r '.param.key' <<< "$item") + metadataJson='{ "uid":"'${parameterUID}'" }' + echo ${metadataJson} > ./metadata.json + echo ${data} > ./${key}_${source_type}.json + configPerEnv . ${envTypes} "project_parameter" ${key}_${source_type}.json ${key} + #cp -n ./${key}_${source_type}.json ${key}_dev.json + #cp -n ./${key}_${source_type}.json ${key}_qa.json + #cp -n ./${key}_${source_type}.json ${key}_prod.json + cd .. + done + fi + echo "Project Parameters export Succeeded" + fi + cd ${HOME_DIR}/${repoName} +} + + if [ ${synchProject} == true ]; then echod "Listing All Assets" echod $assetType @@ -288,7 +389,7 @@ if [ ${synchProject} == true ]; then --header 'Accept: application/json' \ -u ${admin_user}:${admin_password}) - # Exporing APIs + # Exporting APIs for item in $(jq -c -r '.output.rest_api[]' <<< "$projectListJson"); do echod "Inside REST_API Loop" assetID=$item @@ -297,7 +398,7 @@ if [ ${synchProject} == true ]; then exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} done - # Exporing Workflows + # Exporting Workflows for item in $(jq -c -r '.output.workflows[]' <<< "$projectListJson"); do echod "Inside Workflow Loop" assetID=$item @@ -339,34 +440,11 @@ if [ ${synchProject} == true ]; then # Exporting Project Referencedata exportReferenceData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} - - # Exporting Project Parameters - : ' PP Export - PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params - - ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - -u ${admin_user}:${admin_password}) - - ppListExport=$(echo "$ppListJson" | jq '. // empty') - - if [ -z "$ppListExport" ]; then - echo "No Project Parameters retreived:" ${ppListJson} - else - mkdir -p ./assets/projectConfigs/parameters - cd ./assets/projectConfigs/parameters - for item in $(jq -c -r '.output[]' <<< "$ppListJson"); do - echod "Inside Parameters Loop" - parameterUID=$(jq -r '.uid' <<< "$item") - data=$(jq -r '.param' <<< "$item") - echo ${data} > ./${parameterUID}.json - done - echo "Project Parameters export Succeeded" - fi - cd ${HOME_DIR}/${repoName} - ' + #PP Export + assetType=project_parameter + exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} + else exportAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} fi diff --git a/pipelines/scripts/importAsset.sh b/pipelines/scripts/importAsset.sh index 1abefb8..fd0e68c 100644 --- a/pipelines/scripts/importAsset.sh +++ b/pipelines/scripts/importAsset.sh @@ -59,6 +59,7 @@ debug=${@: -1} fi if [ "$debug" == "debug" ]; then echo "......Running in Debug mode ......" + set -x fi @@ -66,7 +67,7 @@ function echod(){ if [ "$debug" == "debug" ]; then echo $1 - set -x + fi } @@ -105,40 +106,66 @@ function importAsset() { importSingleRefData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} ${projectID} fi else - if [[ $assetType = workflow* ]]; then - FLOW_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflow-import - cd ${HOME_DIR}/${repoName}/assets/workflows - echod "Workflow Import:" ${FLOW_URL} + if [[ $assetType = rest_api* ]]; then + IMPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/project-import + cd ${HOME_DIR}/${repoName}/assets/rest_api + echod "REST API Import:" ${IMPORT_URL} echod $(ls -ltr) else - if [[ $assetType = flowservice* ]]; then - FLOW_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flow-import - cd ${HOME_DIR}/${repoName}/assets/flowservices - echod "Flowservice Import:" ${FLOW_URL} - echod $(ls -ltr) + if [[ $assetType = workflow* ]]; then + IMPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/workflow-import + cd ${HOME_DIR}/${repoName}/assets/workflows + echod "Workflow Import:" ${IMPORT_URL} + echod $(ls -ltr) + else + if [[ $assetType = project_parameter* ]]; then + echod "Project Parameter Import:" ${assetID} + importSingleProjectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} ${projectID} + return + else + if [[ $assetType = flowservice* ]]; then + IMPORT_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/flow-import + cd ${HOME_DIR}/${repoName}/assets/flowservices + echod "Flowservice Import:" ${IMPORT_URL} + echod $(ls -ltr) + fi + fi fi - fi - echod ${FLOW_URL} + fi + echod ${IMPORT_URL} echod ${PWD} FILE=./${assetID}.zip - formKey="recipe=@"${FILE} + if [[ $assetType = rest_api* ]]; then + formKey="project=@"${FILE} + else + formKey="recipe=@"${FILE} + fi overwriteKey="overwrite=true" echod ${formKey} if [ -f "$FILE" ]; then ####### Check if asset with this name exist echo "$FILE exists. Importing ..." - importedName=$(curl --location --request POST ${FLOW_URL} \ + importedName=$(curl --location --request POST ${IMPORT_URL} \ --header 'Content-Type: multipart/form-data' \ --header 'Accept: application/json' \ --form ${formKey} --form ${overwriteKey} -u ${admin_user}:${admin_password}) - name=$(echo "$importedName" | jq '.output.name // empty') - if [ -z "$name" ]; then - echo "Import failed:" ${importedName} + if [[ $assetType = rest_api* ]]; then + name=$(echo "$importedName" | jq '.output.message // empty') + success='"IMPORT_SUCCESS"' + if [ "$name" == "$success" ]; then + echo "Import Succeeded:" ${importedName} + else + echo "Import Failed:" ${importedName} + fi else + name=$(echo "$importedName" | jq '.output.name // empty') + if [ -z "$name" ]; then + echo "Import failed:" ${importedName} + else echo "Import Succeeded:" ${importedName} - + fi fi else echo "$FILE does not exists, Nothing to import" @@ -155,6 +182,90 @@ function importAsset() { cd ${HOME_DIR}/${repoName} } +function importSingleProjectParameters(){ + LOCAL_DEV_URL=$1 + admin_user=$2 + admin_password=$3 + repoName=$4 + assetID=$5 + assetType=$6 + HOME_DIR=$7 + synchProject=$8 + source_type=$9 + projectID=${10} + d=$assetID + + cd ${HOME_DIR}/${repoName} + #Importing Reference Data + DIR="./assets/projectConfigs/parameters/" + if [ -d "$DIR" ]; then + echo "Project parameters needs to be synched" + echod "ProjectID:" ${projectID} + cd ./assets/projectConfigs/parameters/ + if [ -d "$d" ]; then + echod "$d" + cd "$d" + if [ ! -f ./metadata.json ]; then + echo "Metadata not found!" + exit 1 + fi + parameterUID=`jq -r '.uid' ./metadata.json | tr -d '\n\t'` + echod "Picked from Metadata: "$parameterUID + + PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${parameterUID} + echod ${PROJECT_PARAM_GET_URL} + ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + -u ${admin_user}:${admin_password}) + ppExport=$(echo "$ppListJson" | jq '.output.uid // empty') + echod ${ppExport} + if [ -z "$ppExport" ]; then + echo "Project parameters does not exists, creating ..:" + PROJECT_PARAM_CREATE_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params + echod ${PROJECT_PARAM_CREATE_URL} + parameterJSON=`jq -c '.' ./*_${source_type}.json` + + echod "Param JSON: "${parameterJSON} + echod "curl --location --request POST ${PROJECT_PARAM_CREATE_URL} \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --data-raw "$parameterJSON" -u ${admin_user}:${admin_password})" + + ppCreateJson=$(curl --location --request POST ${PROJECT_PARAM_CREATE_URL} \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --data-raw "$parameterJSON" -u ${admin_user}:${admin_password}) + ppCreatedJson=$(echo "$ppCreateJson" | jq '.output.uid // empty') + if [ -z "$ppCreatedJson" ]; then + echo "Project Paraters Creation failed:" ${ppCreateJson} + else + echo "Project Paraters Creation Succeeded, UID:" ${ppCreatedJson} + fi + else + echo "Project parameters does exists, updating ..:" + PROJECT_PARAM_UPDATE_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${parameterUID} + echod ${PROJECT_PARAM_UPDATE_URL} + parameterJSON=`jq -c '.' ./*_${source_type}.json` + echod "Param: "${parameterJSON} + ppUpdateJson=$(curl --location --request PUT ${PROJECT_PARAM_UPDATE_URL} \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + -d ${parameterJSON} -u ${admin_user}:${admin_password}) + ppUpdatedJson=$(echo "$ppUpdateJson" | jq '.output.uid // empty') + if [ -z "$ppUpdatedJson" ]; then + echo "Project Paraters Update failed:" ${ppUpdateJson} + else + echo "Project Paraters Update Succeeded, UID:" ${ppUpdatedJson} + fi + fi + else + echo "Invalid Project Parameter / Asset Id to import." + fi + else + echo "No Project Parameters to import." + fi +} function importSingleRefData(){ LOCAL_DEV_URL=$1 @@ -236,6 +347,7 @@ function importRefData(){ source_type=$9 cd ${HOME_DIR}/${repoName} + ls -ltr #Importing Reference Data DIR="./assets/projectConfigs/referenceData/" @@ -254,7 +366,7 @@ function importRefData(){ echod "ProjectID:" ${projectID} cd ./assets/projectConfigs/referenceData/ for d in * ; do - importSingleRefData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${d} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} ${projectID}- + importSingleRefData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${d} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} ${projectID} done fi cd ${HOME_DIR}/${repoName} @@ -280,64 +392,13 @@ function projectParameters(){ if [ -d "$DIR" ]; then echo "Project Parameters needs to be synched" cd ./assets/projectConfigs/parameters/ - for filename in ./*.json; do - parameterUID=${filename##*/} - parameterUID=${parameterUID%.*} - echod ${parameterUID} - PROJECT_PARAM_GET_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${parameterUID} - echod ${PROJECT_PARAM_GET_URL} - ppListJson=$(curl --location --request GET ${PROJECT_PARAM_GET_URL} \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - -u ${admin_user}:${admin_password}) - - ppExport=$(echo "$ppListJson" | jq '.output.uid // empty') - echod ${ppExport} - if [ -z "$ppExport" ]; then - echo "Project parameters does not exists, creating ..:" - PROJECT_PARAM_CREATE_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params - echod ${PROJECT_PARAM_CREATE_URL} - parameterJSON="$(cat ${parameterUID}.json)" - echod "${parameterJSON}" - echod "curl --location --request POST ${PROJECT_PARAM_CREATE_URL} \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - --data-raw "$parameterJSON" -u ${admin_user}:${admin_password})" - - ppCreateJson=$(curl --location --request POST ${PROJECT_PARAM_CREATE_URL} \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - --data-raw "$parameterJSON" -u ${admin_user}:${admin_password}) - ppCreatedJson=$(echo "$ppCreateJson" | jq '.output.uid // empty') - if [ -z "$ppCreatedJson" ]; then - echo "Project Paraters Creation failed:" ${ppCreateJson} - else - echo "Project Paraters Creation Succeeded, UID:" ${ppCreatedJson} - fi - else - echo "Project parameters does exists, updating ..:" - PROJECT_PARAM_UPDATE_URL=${LOCAL_DEV_URL}/apis/v1/rest/projects/${repoName}/params/${parameterUID} - echod ${PROJECT_PARAM_UPDATE_URL} - parameterJSON=`jq '.' ${parameterUID}.json` - echod ${parameterJSON} - ppUpdateJson=$(curl --location --request POST ${PROJECT_PARAM_UPDATE_URL} \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json' \ - -d ${parameterJSON} -u ${admin_user}:${admin_password}) - ppUpdatedJson=$(echo "$ppUpdateJson" | jq '.output.uid // empty') - if [ -z "$ppUpdatedJson" ]; then - echo "Project Paraters Creation failed:" ${ppUpdateJson} - else - echo "Project Paraters Creation Succeeded, UID:" ${ppUpdatedJson} - fi - fi + for d in * ; do + importSingleProjectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${d} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} ${projectID} done else echo "No Project Parameters to import." fi - cd ${HOME_DIR}/${repoName} - } @@ -345,17 +406,51 @@ cd ${HOME_DIR}/${repoName} if [ ${synchProject} == true ]; then echod "Listing files" - for filename in ./assets/*/*.zip; do - base_name=${filename##*/} - parent_name="$(basename "$(dirname "$filename")")" - base_name=${base_name%.*} - echod $base_name${filename%.*} - echod $parent_name - importAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${base_name} ${parent_name} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} - done - + shopt -s nullglob dotglob + api_files=(./assets/rest_api/*.zip) + if [ ${#api_files[@]} -gt 0 ]; then + for filename in ./assets/rest_api/*.zip; do + base_name=${filename##*/} + parent_name="$(basename "$(dirname "$filename")")" + base_name=${base_name%.*} + echod $base_name${filename%.*} + echod $parent_name + importAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${base_name} ${parent_name} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} + done + else + echod "No rest apis to import" + fi + + shopt -s nullglob dotglob + wf_files=(./assets/workflows/*.zip) + if [ ${#wf_files[@]} -gt 0 ]; then + for filename in ./assets/workflows/*.zip; do + base_name=${filename##*/} + parent_name="$(basename "$(dirname "$filename")")" + base_name=${base_name%.*} + echod $base_name${filename%.*} + echod $parent_name + importAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${base_name} ${parent_name} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} + done + else + echod "No workflows to import" + fi + shopt -s nullglob dotglob + fs_files=(./assets/flowservices/*.zip) + if [ ${#fs_files[@]} -gt 0 ]; then + for filename in ./assets/flowservices/*.zip; do + base_name=${filename##*/} + parent_name="$(basename "$(dirname "$filename")")" + base_name=${base_name%.*} + echod $base_name${filename%.*} + echod $parent_name + importAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${base_name} ${parent_name} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} + done + else + echod "No flowservices to import" + fi importRefData ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} - #projectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${base_name} ${parent_name} ${HOME_DIR} ${synchProject} ${source_type} + projectParameters ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${source_type} else importAsset ${LOCAL_DEV_URL} ${admin_user} ${admin_password} ${repoName} ${assetID} ${assetType} ${HOME_DIR} ${synchProject} ${inlcudeAllReferenceData} diff --git a/pipelines/scripts/loadConfig.sh b/pipelines/scripts/loadConfig.sh index 085a776..1f46e0d 100644 --- a/pipelines/scripts/loadConfig.sh +++ b/pipelines/scripts/loadConfig.sh @@ -2,10 +2,10 @@ cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/dev.yml`" - echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/dev.yml`" - echo "##vso[task.setvariable variable=exporter_user]`yq -e ".tenant.exporter_username" configs/dev.yml`" - echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/dev.yml`" + echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=exporter_user]`yq -e ".tenant.exporter_username" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/dev.yml`" PROJECTNAME=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$PROJECTNAME") echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-${PROJECTNAME}" \ No newline at end of file diff --git a/pipelines/scripts/readEnvs.sh b/pipelines/scripts/readEnvs.sh new file mode 100644 index 0000000..129d2c6 --- /dev/null +++ b/pipelines/scripts/readEnvs.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +############################################################################# +# # +# readEnvs.sh : read number of environments from the congig directory # +# # +############################################################################# + + +CONFIG_DIR=$1 +source_type=$2 +debug=${@: -1} + + if [ -z "$CONFIG_DIR" ]; then + echo "Missing template parameter CONFIG_DIR" + exit 1 + fi + + if [ -z "$source_type" ]; then + echo "Missing template parameter source_type" + exit 1 + fi + + if [ "$debug" == "debug" ]; then + set -x + echo "......Running in Debug mode ......" + fi + + +function echod(){ + if [ "$debug" == "debug" ]; then + echo $1 + fi + +} +declare -a envArr +i=0 +envs='' +shopt -s nullglob dotglob +env_files=($CONFIG_DIR/*.yml) +if [ ${#env_files[@]} -gt 0 ]; then + for env in $CONFIG_DIR/*.yml; do + echod $env + current_type=$(cat $env | yq -e '.tenant.type') + if [ ${current_type} != ${source_type} ]; then + envArr[i]=$(cat $env | yq -e '.tenant.type') + echod $envArr[i] + let i++ + fi + done + envs=$(IFS=$','; echo "${envArr[*]}") + + echod $envs +else + echo "No environment file found" + exit 1 +fi + +echo $envs + +set +x + + diff --git a/pipelines/sychronizeFeatureBranch_pipeline.yml b/pipelines/sychronizeFeatureBranch_pipeline.yml index 9defed7..6294d35 100644 --- a/pipelines/sychronizeFeatureBranch_pipeline.yml +++ b/pipelines/sychronizeFeatureBranch_pipeline.yml @@ -46,12 +46,12 @@ steps: cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/play.yml`" - echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/play.yml`" - echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/play.yml`" - echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/play.yml`" + echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env/play.yml`" + echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/play.yml`" + echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/play.yml`" + echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/play.yml`" PROJECTNAME=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$(repoName)") - echo ${PROJECTNAME} + #echo ${PROJECTNAME} echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-${PROJECTNAME}" echo $(System.DefaultWorkingDirectory) workingDirectory: '$(System.DefaultWorkingDirectory)' @@ -72,11 +72,18 @@ steps: cd $(repoName) chmod +x ../self/pipelines/scripts/*.sh + config_dir=${HOME_DIR}self/configs/env + + + ###### Fetching Environments + echo "Fetching Environment Types ..." + envTypes=$(../self/pipelines/scripts/readEnvs.sh ${config_dir} $(source_type)) + echo "****Envs****: "$envTypes ###### Exporting echo "Exporting asset ..." - ../self/pipelines/scripts/exportAsset.sh ${LOCAL_DEV_URL} $(admin_user) $(admin_password) $(repoName) $(assetID) $(assetType) ${HOME_DIR} $(synchProject) $(source_type) $(inlcudeAllReferenceData) debug + ../self/pipelines/scripts/exportAsset.sh ${LOCAL_DEV_URL} $(admin_user) $(admin_password) $(repoName) $(assetID) $(assetType) ${HOME_DIR} $(synchProject) $(source_type) $(inlcudeAllReferenceData) ${envTypes} debug displayName: 'Export Asset' #failOnStderr: true @@ -86,7 +93,7 @@ steps: echo "Committing asset ..." chmod +x ../self/pipelines/scripts/github/*.sh - ../self/pipelines/scripts/github/commitFeatureBranch.sh $(devUser) $(Build.BuildNumber) $(featureBranchName) ${HOME_DIR} debugebug + ../self/pipelines/scripts/github/commitFeatureBranch.sh $(devUser) $(Build.BuildNumber) $(featureBranchName) ${HOME_DIR} debug displayName: 'Commit the code to feature branch' diff --git a/pipelines/sychronizeToQA_pipeline.yml b/pipelines/sychronizeToQA_pipeline.yml index b59207a..9c62d69 100644 --- a/pipelines/sychronizeToQA_pipeline.yml +++ b/pipelines/sychronizeToQA_pipeline.yml @@ -54,10 +54,10 @@ steps: cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/qa.yml`" - echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/qa.yml`" - echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/qa.yml`" - echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/qa.yml`" + echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env//qa.yml`" + echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/qa.yml`" + echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/qa.yml`" + echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/qa.yml`" repoName=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$(repoName)") # echo $(repoName) echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-$(repoName)" diff --git a/pipelines/synchronizeDEV_pipeline.yml b/pipelines/synchronizeDEV_pipeline.yml index bc263aa..52c7d5a 100644 --- a/pipelines/synchronizeDEV_pipeline.yml +++ b/pipelines/synchronizeDEV_pipeline.yml @@ -54,10 +54,10 @@ steps: cd ../self pwd ls -ltr - echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/dev.yml`" - echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/dev.yml`" - echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/dev.yml`" - echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/dev.yml`" + echo "##vso[task.setvariable variable=source_environment_hostname]`yq -e ".tenant.hostname" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=source_environment_port]`yq -e ".tenant.port" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=admin_user]`yq -e ".tenant.admin_username" configs/env/dev.yml`" + echo "##vso[task.setvariable variable=source_type]`yq -e ".tenant.type" configs/env/dev.yml`" repoName=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$(repoName)") # echo $(repoName) echo "##vso[build.updatebuildnumber]$(Build.BuildNumber)-$(repoName)"