Update the config file; Update the get_modified_packs file; Convert to poetry

.github/workflows/config.yml

@@ -1,99 +1,73 @@
 name: XSOAR CI/CD
 on:
   push:
+
 jobs:
   build:
     runs-on: ubuntu-latest
+    env:
+      BRANCH_NAME: ${{ github.ref_name }}
+      DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+      DEMISTO_README_VALIDATION: false
+      DEMISTO_SDK_GITHUB_TOKEN: ${{ secrets.DEMISTO_SDK_GITHUB_TOKEN }}
     steps:
       - name: XSOAR CI/CD master checkout
         uses: actions/checkout@v2
         with:
-          repository: your/repository
-          path: repository
           fetch-depth: 0
       - name: Content checkout
         uses: actions/checkout@v2
         with:
           repository: demisto/content
           path: content
+      - name: Install poetry
+        uses: Gr1N/setup-poetry@v8
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8.5
-      - name: Install pip and requirements
+          python-version: 3.9.12
+          cache: poetry
+      - name: Install python dependencies
         run: |
-          pip install --upgrade pip
-          pip install -r ${GITHUB_WORKSPACE}/repository/requirements.txt
+          poetry install
       - name: Prepare Environment
-        env:
-          REF: ${{ github.ref }}
         run: |
           echo "PATH=/home/runner/.local/bin:$PATH" >> $GITHUB_ENV
           echo "PYTHONPATH=${GITHUB_WORKSPACE}/content:${GITHUB_WORKSPACE}:${PYTHONPATH}" >> $GITHUB_ENV
-          echo "ARTIFACTS_FOLDER=/home/runner/work/repository/repository/content/artifacts" >> $GITHUB_ENV
-          echo "NEW_PACKS_FOLDER=/home/runner/work/repository/repository/content/new_packs_zips" >> $GITHUB_ENV
-          echo "DEMISTO_README_VALIDATION=false" >> $GITHUB_ENV
+          echo "NEW_PACKS_FOLDER=${GITHUB_WORKSPACE}/content/new_packs_zips" >> $GITHUB_ENV
 
-          BRANCH_NAME=${REF#refs/heads/}
-          PACKS_CHANGED=$(python $GITHUB_WORKSPACE/repository/build_related_scripts/get_modified_packs.py --repo_path $GITHUB_WORKSPACE/repository/)
+          PACKS_CHANGED=$(python $GITHUB_WORKSPACE/build_related_scripts/get_modified_packs.py --repo_path $GITHUB_WORKSPACE/ --prev-ver ${DEFAULT_BRANCH})
           echo "PACKS_CHANGED=${PACKS_CHANGED}" >> $GITHUB_ENV
-          echo "DEMISTO_SDK_GITHUB_TOKEN=<your-github-token>" >> $GITHUB_ENV
+
+          echo "The current branch is ${BRANCH_NAME}"
+          echo "The default branch is ${DEFAULT_BRANCH}"
 
       - name: Prepare Venv
         run: |
           cd $GITHUB_WORKSPACE/content/
 
-          if [ ! -e "venv" ]; then
-            echo "installing venv"
-            NO_HOOKS=1 SETUP_PY2=no .hooks/bootstrap
-            source ./venv/bin/activate
-            pip3 install -r ${GITHUB_WORKSPACE}/repository/requirements.txt
-            demisto-sdk --version
-          else
-            echo "venv exists (from cache). activating"
-            source ./venv/bin/activate
-          fi
-
           # Install node packages
           npm install .
           git config diff.renameLimit 6000
           echo "========== Build Parameters =========="
-          python3 --version
+          python --version
           demisto-sdk --version
-      - name: Create ID Set
-        run: |
-          if [ $PACKS_CHANGED ]; then
-            cd $GITHUB_WORKSPACE/repository/
-            source $GITHUB_WORKSPACE/content/venv/bin/activate
-
-            # Download content's
-            gsutil cp gs://marketplace-dist/content/id_set.json ./Tests/content-id_set.json
-
-            # Create repository id-set
-            demisto-sdk create-id-set
-
-            # Merge id-sets and copy to artifacts folder
-            demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 ./Tests/content-id_set.json -o ./Tests/id_set.json
-            cp "./Tests/id_set.json" "$ARTIFACTS_FOLDER/id_set.json"
-          else
-            echo "No packs has changed, skipping step."
-          fi
       - name: Validate Files and Yaml
         if: always()
         run: |
-          source $GITHUB_WORKSPACE/content/venv/bin/activate
+          source $GITHUB_WORKSPACE/content/.venv/bin/activate
 
-          cd $GITHUB_WORKSPACE/repository
+          cd $GITHUB_WORKSPACE
 
           # Run validate on all changed files
           # You can choose what validations to use in the .demisto-sdk-conf file
-          demisto-sdk validate --quite-bc-validation --no-conf-json --allow-skipped
+          demisto-sdk validate --use-git --post-commit --graph --prev-ver ${DEFAULT_BRANCH}
       - name: Run Unit Testing and Lint
         if: always()
         run: |
-          source $GITHUB_WORKSPACE/content/venv/bin/activate
+          source $GITHUB_WORKSPACE/content/.venv/bin/activate
 
-          cd $GITHUB_WORKSPACE/repository
+          cd $GITHUB_WORKSPACE
 
           # Copy necessary files
           cp $GITHUB_WORKSPACE/content/Tests/demistomock/demistomock.py ./
@@ -105,52 +79,56 @@ jobs:
 
           # Run lint on all changed files
           # You can choose what lint checks to use in the .demisto-sdk-conf file
-          demisto-sdk lint -g
+          demisto-sdk lint -g --prev-ver ${DEFAULT_BRANCH}
       - name: Create Packs Artifacts
         run: |
           if [ $PACKS_CHANGED ]; then
-            source $GITHUB_WORKSPACE/content/venv/bin/activate
+            source $GITHUB_WORKSPACE/content/.venv/bin/activate
 
-            cd $GITHUB_WORKSPACE/repository/
+            cd $GITHUB_WORKSPACE/
             for pack in Packs/*
             do
-              demisto-sdk zip-packs --input $pack --output $ARTIFACTS_FOLDER
+              demisto-sdk prepare-content --input $pack --output $NEW_PACKS_FOLDER
             done
           else
             echo "No packs has changed, skipping step."
           fi
       - name: Upload Packs to Artifacts Server
+        # In case 2 below (UPLOAD DIRECTLY TO YOUR XSOAR MACHINE) needed to add these variables
+        # For more information - https://docs-cortex.paloaltonetworks.com/r/1/Demisto-SDK-Guide/Environment-variables-setup
         env:
-          REF: ${{ github.ref }}
+          DEMISTO_BASE_URL: ${{ secrets.DEMISTO_BASE_URL }}
+          DEMISTO_API_KEY: ${{ secrets.DEMISTO_API_KEY }}
+          # For Cortex XSOAR 8 and Cortex XSIAM add the following variable:
+          XSIAM_AUTH_ID: ${{ vars.XSIAM_AUTH_ID }}
         run : |
           if [ $PACKS_CHANGED ]; then
-            BRANCH_NAME=${REF#refs/heads/}
+            if [ $BRANCH_NAME == ${DEFAULT_BRANCH} ]; then
+              echo "Uploading artifacts ${PACKS_CHANGED}."
 
-            # Select the way you want to install the packs on the machine, and remove the second part
-            # For example: Choose the option 1 - UPLOAD TO ARTIFACTS SERVER OPTION, and remove the option 2 - UPLOAD DIRECTLY TO YOUR XSOAR MACHINE.
-            # For more information - https://xsoar.pan.dev/docs/reference/articles/xsoar-ci-cd#to-deploy-content
+              # Select the way you want to install the packs on the machine, and remove the second part
+              # For example: Choose the option 1 - UPLOAD TO ARTIFACTS SERVER OPTION, and remove the option 2 - UPLOAD DIRECTLY TO YOUR XSOAR MACHINE.
+              # For more information - https://xsoar.pan.dev/docs/reference/articles/xsoar-ci-cd#to-deploy-content
 
-            # 1. ========= UPLOAD TO ARTIFACTS SERVER OPTION =========
+              # 1. ========= UPLOAD TO ARTIFACTS SERVER OPTION =========
 
-            # Upload to the artifacts server of your choice.
-            # Create a file with the service account data
-            # use the bucket_upload script to upload your packs to google cloud storage
-            python $GITHUB_WORKSPACE/repository/build_related_scripts/bucket_upload.py --service_account $GITHUB_WORKSPACE/service_account.json --packs_directory $NEW_PACKS_FOLDER --branch_name $BRANCH_NAME
-            # Delete the service account file
-            rm $GITHUB_WORKSPACE/service_account.json
+              # Upload to the artifacts server of your choice.
+              # Create a file with the service account data
+              # use the bucket_upload script to upload your packs to google cloud storage
+              python $GITHUB_WORKSPACE/build_related_scripts/bucket_upload.py --service_account $GITHUB_WORKSPACE/service_account.json --packs_directory $NEW_PACKS_FOLDER --branch_name $BRANCH_NAME
+              # Delete the service account file
+              rm $GITHUB_WORKSPACE/service_account.json
 
-            # 2. ========= UPLOAD DIRECTLY TO YOUR XSOAR MACHINE (WHEN MERGING TO MAIN REPO) =========
+              # 2. ========= UPLOAD DIRECTLY TO YOUR XSOAR MACHINE (WHEN MERGING TO MAIN REPO) =========
 
-            if [ $BRANCH_NAME != master ]; then
+              cd $NEW_PACKS_FOLDER
+              for pack in *
+              do
+                demisto-sdk upload --input $pack
+              done
 
-              # Get the config_file
-              CONFIG_FILE=$(cat xsoar_config.json)
-              # Extract the Marketplace Packs section from the config_file
-              MARKETPLACE_PACKS_LIST=$(cat $CONFIG_FILE | jq -r '.marketplace_packs')
-              # Upload Custom Packs
-              demisto-sdk upload --input-config-file /xsoar_config.json
-              # Upload MarketPlace Packs
-              python3 build_related_scripts/MarketPlaceInstallerFromCICD.py --marketplace-packs-list $MARKETPLACE_PACKS_LIST
+            else
+              echo "The currrent branch is not the default branch, skipping upload to server."
             fi
 
           else

build_related_scripts/get_modified_packs.py

@@ -5,18 +5,18 @@
 
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.tools import get_pack_names_from_files
-from git import Repo
 
-PACK_PATH_REGEX = r'Packs/([a-zA-Z0-9_]+)/'
+PACK_PATH_REGEX = r"Packs/([a-zA-Z0-9_]+)/"
+
+PACKS = "Packs"
 
 
 def dir_path(path: str):
-    """Directory type module for argparse.
-    """
+    """Directory type module for argparse."""
     if os.path.isdir(path):
         return Path(path)
     else:
-        raise argparse.ArgumentTypeError(f'{path} is not a valid path.')
+        raise argparse.ArgumentTypeError(f"{path} is not a valid path.")
 
 
 def option_handler() -> argparse.Namespace:
@@ -26,9 +26,13 @@ def option_handler() -> argparse.Namespace:
         Namespace: Parsed arguments object.
 
     """
-    parser = argparse.ArgumentParser(description='Collect the packs that has changed.')
-    parser.add_argument('-rp', '--repo_path', help='The path to the required repo.', type=dir_path)
-    parser.add_argument('--prev-ver', default='master', help='Previous branch or SHA1 commit to run checks against.')
+    parser = argparse.ArgumentParser(description="Collect the packs that has changed.")
+    parser.add_argument(
+        "-rp", "--repo_path", help="The path to the required repo.", type=dir_path
+    )
+    parser.add_argument(
+        '--prev-ver', default='master', help='Previous branch or SHA1 commit to run checks against.'
+    )
     return parser.parse_args()
 
 
@@ -41,13 +45,17 @@ def get_changed_files(repo_path: Path, prev_ver: str) -> List[str]:
     Returns:
         List[str]. All the files that have changed.
     """
-    repo = Repo(repo_path, search_parent_directories=True)
     git_util = GitUtil(repo_path)
+    repo = git_util.repo
+
+    try:
+        active_branch = repo.active_branch
+    except TypeError:
+        active_branch = 'DETACHED_' + repo.head.object.hexsha
 
-    if str(repo.active_branch) == prev_ver:
+    if str(active_branch) == prev_ver:
         # Get the latest commit in master, prior the merge.
-        commits_list = list(repo.iter_commits())
-        prev_ver = str(commits_list[1])
+        prev_ver = str(repo.remote().refs[prev_ver].commit.parents[0])
 
     modified_files = git_util.modified_files(prev_ver=prev_ver)
     added_files = git_util.added_files(prev_ver=prev_ver)
@@ -66,10 +74,11 @@ def main():
     changed_files = get_changed_files(repo_path, prev_ver)
 
     packs_changed = get_pack_names_from_files(changed_files)
-    changed_packs_string = ",".join(packs_changed)
+    packs_changed_paths = [str(repo_path / PACKS / pack) for pack in packs_changed]
+    changed_packs_paths_string = ",".join(packs_changed_paths)
 
-    print(changed_packs_string)
+    print(changed_packs_paths_string)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "content-ci-cd-template"
+version = "0.1.0"
+description = ""
+authors = []
+
+[tool.poetry.dependencies]
+python = "^3.8,<3.11"
+demisto-sdk = "*"
+virtualenv = "^20.25.0"
+jsonschema = "^4.21.1"
+gsutil = "^5.27"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"