Merge pull request #10 from arenadata/2.0.1-sync

ADBDEV-3076 2.0.1 sync
arenadata · Oct 10, 2022 · 7cb464b · 7cb464b
2 parents 5835af2 + ef56dda
commit 7cb464b
Show file tree

Hide file tree

Showing 14 changed files with 283 additions and 99 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -97,7 +97,7 @@ set_target_properties(
   PROPERTIES OUTPUT_NAME ${DISKQUOTA_BINARY_NAME}
              PREFIX ""
              C_STANDARD 99
-             LINKER_LANGUAGE "CXX")
+             LINKER_LANGUAGE "C")
 
 # packing part, move to a separate file if this part is too large
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Distro.cmake)
@@ -146,7 +146,7 @@ add_custom_target(create_artifact
   COMMAND
   ${CMAKE_COMMAND} --build . --target package
   COMMAND
-  ${CMAKE_COMMAND} -E tar cvf ${artifact_NAME} "${tgz_NAME}.tar.gz")
+  ${CMAKE_COMMAND} -E tar czvf ${artifact_NAME} "${tgz_NAME}.tar.gz")
 # packing end
 
 # Create build-info

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.0.0
+2.0.1
diff --git a/concourse/README.md b/concourse/README.md
@@ -2,20 +2,21 @@
 
 ## Naming Prefix Rule
 
-- `PR:<project_name>` for pull-request pipelines
-- `COMMIT:<project_name>:<branch_name>` for branch pipelines. It will be executed when a commit committed/merged into the branch.
-- `DEV:<your_name>_<project_name>[any_other_info]` for personal development usage. Put your name into the pipeline name so others can know who own it.
+- `pr.<project_name>` for pull-request pipelines
+- `merge.<project_name>.<branch_name>` for branch pipelines. It will be executed when a commit committed/merged into the branch.
+- `dev.<project_name>.<branch_name>.<your_postfix>` for personal development usage. Put your name into the pipeline name so others can know who own it.
+- `<pipeline>_test.<project_name>.<branch_name>` for pipeline debugging.
 
 ## Pipelines for daily work
 
 ### PR Pipeline
 
-https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/PR:diskquota
+https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/pr.diskquota
 
 ### Main Branch Pipeline
 
-The development happens on the `gpdb` branch. The commit pipeline for the `gpdb`
-https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/COMMIT:diskquota:gpdb
+The development happens on the `gpdb` branch. The merge pipeline for the `gpdb` branch is
+https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/merge.diskquota:gpdb
 
 
 # Fly a pipeline
@@ -38,10 +39,10 @@ https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/COMMIT:diskquota:gpdb
 ./fly.sh -t extension -c pr
 ```
 
-## Fly the commit pipeline
+## Fly the merge pipeline
 
 ```
-./fly.sh -t extension -c commit
+./fly.sh -t extension -c merge
 ```
 
 ## Fly the release pipeline
@@ -54,24 +55,24 @@ The release pipeline should be located in https://prod.ci.gpdb.pivotal.io
 # Login to prod
 fly -t prod login -c https://prod.ci.gpdb.pivotal.io
 # Fly the release pipeline
-./fly.sh -t prod -c release
+./fly.sh -t prod -c rel
 ```
 
 To fly a release pipeline from a specific branch:
 
 ```
-./fly.sh -t <target> -c release -b release/<major>.<minor>
+./fly.sh -t <target> -c rel -b release/<major>.<minor>
 ```
 
 ## Fly the dev pipeline
 
 ```
-./fly.sh -t extension -c dev -p <your_name>_diskquota -b <your_branch>
+./fly.sh -t extension -c dev -p <your_postfix> -b <your_branch>
 ```
 
 ## Webhook
 
-By default, the PR and commit pipelines are using webhook instead of polling to trigger a build. The webhook URL will be printed when flying such a pipeline by `fly.sh`. The webhook needs to be set in the `github repository` -> `Settings` -> `Webhooks` with push notification enabled.
+By default, the PR and merge pipelines are using webhook instead of polling to trigger a build. The webhook URL will be printed when flying such a pipeline by `fly.sh`. The webhook needs to be set in the `github repository` -> `Settings` -> `Webhooks` with push notification enabled.
 
 To test if the webhook works, use `curl` to send a `POST` request to the hook URL with some random data. If it is the right URL, the relevant resource will be refreshed on the Concourse UI. The command line looks like:
 
@@ -83,6 +84,6 @@ curl --data-raw "foo" <hook_url>
 
 ## PR pipeline is not triggered.
 
-The PR pipeline relies on the webhook to detect the new PR. However, due the the limitation of the webhook implemention of concourse, we rely on the push hook for this. It means if the PR is from a forked repo, the PR pipeline won't be triggered immediately. To manually trigger the pipeline, go to https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/PR:diskquota/resources/diskquota_pr and click ⟳ button there.
+The PR pipeline relies on the webhook to detect the new PR. However, due to the the limitation of the webhook implemention of concourse, we rely on the push hook for this. It means if the PR is from a forked repo, the PR pipeline won't be triggered immediately. To manually trigger the pipeline, go to https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/pr.diskquota/resources/diskquota_pr and click ⟳ button there.
 
 TIPS: Just don't fork, name your branch as `<your_id>/<branch_name>` and push it here to create PR.
diff --git a/concourse/fly.sh b/concourse/fly.sh
@@ -2,25 +2,28 @@
 
 set -e
 
-workspace=${WORKSPACE:-"$HOME/workspace"}
 fly=${FLY:-"fly"}
-echo "'workspace' location: ${workspace}"
 echo "'fly' command: ${fly}"
 echo ""
+proj_name="diskquota"
 
 usage() {
-    echo "Usage: $0 -t <concourse_target> -c <pr|commit|dev> [-p <pipeline_name>] [-b branch]" 1>&2
     if [ -n "$1" ]; then
-        echo "$1"
+        echo "$1" 1>&2
+        echo "" 1>&2
     fi
+
+    echo "Usage: $0 -t <concourse_target> -c <pr|rel|merge|dev> [-p <postfix>] [-b branch] [-T]"
+    echo "Options:"
+    echo "       '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging."
     exit 1
 }
 
 # Parse command line options
-while getopts ":c:t:p:b:" o; do
+while getopts ":c:t:p:b:T" o; do
     case "${o}" in
         c)
-            # pipeline type/config. pr/commit/dev/release
+            # pipeline type/config. pr/merge/dev/rel
             pipeline_config=${OPTARG}
             ;;
         t)
@@ -29,12 +32,15 @@ while getopts ":c:t:p:b:" o; do
             ;;
         p)
             # pipeline name
-            pipeline_name=${OPTARG}
+            postfix=${OPTARG}
             ;;
         b)
             # branch name
             branch=${OPTARG}
             ;;
+        T)
+            test_suffix="_test"
+            ;;
         *)
             usage ""
             ;;
@@ -46,52 +52,66 @@ if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then
     usage ""
 fi
 
+pipeline_type=""
 # Decide ytt options to generate pipeline
 case ${pipeline_config} in
   pr)
-      if [ -z "${pipeline_name}" ]; then
-          pipeline_name="PR:diskquota"
-      fi
+      pipeline_type="pr"
       config_file="pr.yml"
-      hook_res="diskquota_pr"
+      hook_res="${proj_name}_pr"
     ;;
-  commit)
-      if [ -z "${pipeline_name}" ]; then
-          pipeline_name="COMMIT:diskquota:gpdb"
-      fi
+  merge|commit)
       # Default branch is 'gpdb' as it is our main branch
       if [ -z "${branch}" ]; then
           branch="gpdb"
       fi
+      pipeline_type="merge"
       config_file="commit.yml"
-      hook_res="diskquota_commit"
+      hook_res="${proj_name}_commit"
     ;;
   dev)
-      if [ -z "${pipeline_name}" ]; then
-          usage "'-p' needs to be supplied to specify the pipeline name for flying a 'dev' pipeline."
+      if [ -z "${postfix}" ]; then
+          usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline."
+      fi
+      if [ -z "${branch}" ]; then
+          usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline."
       fi
-      pipeline_name="DEV:${pipeline_name}"
+      pipeline_type="dev"
       config_file="dev.yml"
     ;;
-  release)
+  release|rel)
       # Default branch is 'gpdb' as it is our main branch
       if [ -z "${branch}" ]; then
           branch="gpdb"
       fi
-      if [ -z "${pipeline_name}" ]; then
-          pipeline_name="RELEASE:diskquota:${branch}"
-      fi
+      pipeline_type="rel"
       config_file="release.yml"
-      hook_res="diskquota_commit"
+      hook_res="${proj_name}_commit"
     ;;
   *)
       usage ""
     ;;
 esac
 
-yml_path="/tmp/diskquota_pipeline.yml"
+yml_path="/tmp/${proj_name}.yml"
 my_path=$(realpath "${BASH_SOURCE[0]}")
 ytt_base=$(dirname "${my_path}")/pipeline
+# pipeline cannot contain '/'
+pipeline_name=${pipeline_name/\//"_"}
+
+# Generate pipeline name
+if [ -n "${test_suffix}" ]; then
+    pipeline_type="${pipeline_type}_test"
+fi
+pipeline_name="${pipeline_type}.${proj_name}"
+if [ -n "${branch}" ]; then
+    pipeline_name="${pipeline_name}.${branch}"
+fi
+if [ -n "${postfix}" ]; then
+    pipeline_name="${pipeline_name}.${postfix}"
+fi
+# pipeline cannot contain '/'
+pipeline_name=${pipeline_name/\//"_"}
 
 ytt --data-values-file "${ytt_base}/res_def.yml" \
     -f "${ytt_base}/base.lib.yml" \
@@ -108,10 +128,7 @@ set -v
     sp \
     -p "${pipeline_name}" \
     -c "${yml_path}" \
-    -l "${workspace}/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml" \
-    -l "${workspace}/gp-continuous-integration/secrets/gp-extensions-common.yml" \
-    -l "${workspace}/gp-continuous-integration/secrets/gpdb_6X_STABLE-ci-secrets.prod.yml" \
-    -v "diskquota-branch=${branch}"
+    -v "${proj_name}-branch=${branch}"
 set +v
 
 if [ "${pipeline_config}" == "dev" ]; then

diff --git a/concourse/pipeline/job_def.lib.yml b/concourse/pipeline/job_def.lib.yml
@@ -1,14 +1,22 @@
 #@ load("base.lib.yml", "add_res_by_conf", "add_res_by_name")
 #@ load("@ytt:template", "template")
 
+#@ def inter_bin_name(base_name, release_build):
+#@   if release_build:
+#@     return base_name + "_rel"
+#@   end
+#@
+#@   return base_name
+#@ end
+
 #! Job config for centos6
 #! Use bin_gpdb_postfix="" to use a release version of gpdb binary
 #@ def centos6_gpdb6_conf(release_build=False):
 res_build_image: centos6-gpdb6-image-build
 res_test_image: centos6-gpdb6-image-test
 res_gpdb_bin: #@ "bin_gpdb6_centos6" + ("" if release_build else "_debug")
 res_diskquota_bin: bin_diskquota_gpdb6_rhel6
-res_intermediates_bin: bin_diskquota_gpdb6_rhel6_intermediates
+res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel6_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel6_release
 os: rhel6
 build_type: #@ "Release" if release_build else "Debug"
@@ -20,7 +28,7 @@ res_build_image: centos7-gpdb6-image-build
 res_test_image: centos7-gpdb6-image-test
 res_gpdb_bin: #@ "bin_gpdb6_centos7" + ("" if release_build else "_debug")
 res_diskquota_bin: bin_diskquota_gpdb6_rhel7
-res_intermediates_bin: bin_diskquota_gpdb6_rhel7_intermediates
+res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel7_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel7_release
 os: rhel7
 build_type: #@ "Release" if release_build else "Debug"
@@ -32,7 +40,7 @@ res_build_image: rhel8-gpdb6-image-build
 res_test_image: rhel8-gpdb6-image-test
 res_gpdb_bin: #@ "bin_gpdb6_rhel8" + ("" if release_build else "_debug")
 res_diskquota_bin: bin_diskquota_gpdb6_rhel8
-res_intermediates_bin: bin_diskquota_gpdb6_rhel8_intermediates
+res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel8_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel8_release
 os: rhel8
 build_type: #@ "Release" if release_build else "Debug"
@@ -44,7 +52,7 @@ res_build_image: ubuntu18-gpdb6-image-build
 res_test_image: ubuntu18-gpdb6-image-test
 res_gpdb_bin: #@ "bin_gpdb6_ubuntu18" + ("" if release_build else "_debug")
 res_diskquota_bin: bin_diskquota_gpdb6_ubuntu18
-res_intermediates_bin: bin_diskquota_gpdb6_ubuntu18_intermediates
+res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_ubuntu18_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_ubuntu18_release
 os: ubuntu18.04
 build_type: #@ "Release" if release_build else "Debug"
@@ -153,9 +161,12 @@ plan:
 #@   end
 - in_parallel:
     steps:
-#@   for conf in confs:
+#@   for i in range(len(confs)):
+#@     conf = confs[i]
       - do:
         - get: #@ conf["res_intermediates_bin"]
+          passed:
+            - #@ passed_jobs[i]
           params:
             unpack: true
         - put: #@ conf["release_bin"]