Cisco WSA

.devcontainer/Dockerfile

@@ -1,34 +1,33 @@
 FROM python:3.10-slim-bullseye
 
-ADD createCerts.sh .
-RUN apt-get update && apt-get install dos2unix -y \
-    && dos2unix /createCerts.sh \
-    && chmod +x /createCerts.sh \ 
-    && /createCerts.sh /usr/local/share/ca-certificates/certs.crt \
-    && update-ca-certificates \
-    && apt-get install python2 -y
-
 ENV USERNAME demisto
 ENV HOME /home/$USERNAME
 ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt
 ENV PATH $PATH:$HOME/.local/bin:/root/.local/bin:/usr/local/share/nvm/current/bin
 
-# This is a workaround for VSCode devcontainer features in self signed certificate
-
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/common-utils/install.sh /install-common.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/git/install.sh /install-git.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/docker-in-docker/install.sh /install-dind.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/node/install.sh /install-node.sh
+ADD createCerts.sh .
+RUN apt-get update && apt-get install dos2unix git python2 curl -y \
+    && dos2unix /createCerts.sh \
+    && chmod +x /createCerts.sh \ 
+    && /createCerts.sh $NODE_EXTRA_CA_CERTS \
+    && update-ca-certificates \
+    && rm -rf /features \
+    && git clone https://github.com/devcontainers/features.git /features \
+    && cd /features \
+    # locking to the latest master commit in this repo to prevent breaking changes
+    # We should update this commit hash from time to time to
+    && git checkout 42e5a013cc914f87d0ff7bf613a1f80d9ff57579
 
-RUN UID="1000" GID="1000" bash install-common.sh 
-RUN VERSION="os-provided" bash install-git.sh
-RUN VERSION="latest" bash install-dind.sh
-RUN VERSION="lts" bash install-node.sh
+# This is a workaround for VSCode devcontainer features in self signed certificate
+RUN UID="1000" GID="1000" bash /features/src/common-utils/install.sh
+RUN VERSION="os-provided" bash /features/src/git/install.sh
+RUN VERSION="latest" bash /features/src/docker-in-docker/install.sh
+RUN VERSION="lts" bash /features/src/node/install.sh
 
 # install poetry, configure certificate for git and tools for oh my zsh
-RUN curl -sSL https://install.python-poetry.org | python3 - \
-    && python -m pip install --user pipx \
+RUN python -m pip install --user pipx \
     && python -m pipx ensurepath \
+    && pipx install poetry \
     && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $HOME/.zsh/zsh-syntax-highlighting \
     && echo "source ~/.zsh/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" >> $HOME/.zshrc \
     && git clone https://github.com/zsh-users/zsh-autosuggestions $HOME/.zsh/zsh-autosuggestions \

.devcontainer/createCerts.sh

@@ -5,6 +5,13 @@
 # It tries to connect to a server (for instance, github), show the local certificates and save it to a file.
 
 # We connect to a random server and not paloaltonetworks.com to get external certificates. 
+# if command fails 
+
+if curl -I https://github.com > /dev/null; then
+    echo "No need to update certificate"
+    exit
+fi
+
 CONNECT_SERVER="github.com:443"
 
 FILE=$1
@@ -21,3 +28,6 @@ if [ ! -f "$FILE" ]; then
     exit
 fi
 
+git config --system http.sslCAInfo $FILE
+
+

.devcontainer/createCommand.sh

@@ -6,22 +6,19 @@ echo "Fixing permissions"
 
 sudo chown demisto .venv
 sudo chown demisto node_modules
+sudo chown demisto /workspaces
 sudo chown -R demisto $HOME
-sudo chown -R demisto /workspaces
-
-echo "Setting up git certificate"
-
-sudo git config --system http.sslCAInfo /usr/local/share/ca-certificates/certs.crt
 
 echo "Setting up VSCode paths"
 
 cp .devcontainer/settings.json .vscode/settings.json 
 touch CommonServerUserPython.py
+touch DemistoClassApiModule.py
 path=$(printf '%s:' Packs/ApiModules/Scripts/*)
 rm -f .env
 echo "PYTHONPATH=""$path"":$PYTHONPATH" >> .env
 echo "MYPYPATH=""$path"":$MYPYPATH" >> .env
 
 echo "Setting up content dependencies"
-
-NO_HOOKS=1 .hooks/bootstrap
+poetry install
+npm install

.devcontainer/devcontainer.json

@@ -14,49 +14,8 @@
 		"source=node-modules,target=${containerWorkspaceFolder}/node_modules,type=volume",
 		"source=dind-var-lib-docker,target=/var/lib/docker,type=volume"
 	],
+	"containerUser": "demisto",
 	"remoteUser": "demisto",
-	"settings": {
-		"terminal.integrated.defaultProfile.linux": "zsh",
-		"terminal.integrated.profiles.linux": {
-			"zsh": {
-				"path": "/bin/zsh"
-			}
-		},
-		"cSpell.words": [
-			"demisto",
-			"xsoar",
-			"xsiam",
-			"fromversion",
-			"toversion",
-			"marketplacev",
-			"ciac",
-			"whois",
-		]
-	},
-	"extensions": [
-		"cortexxsoarext.xsoar",
-		"ms-python.python",
-		"ms-vscode.PowerShell",
-		"ms-toolsai.jupyter",
-		"timonwong.shellcheck",
-		"GitHub.vscode-pull-request-github",
-		"eamodio.gitlens",
-		"ms-azuretools.vscode-docker",
-		"ms-vscode-remote.remote-containers",
-		"streetsidesoftware.code-spell-checker",
-		"njpwerner.autodocstring",
-		"VisualStudioExptTeam.vscodeintellicode",
-		"yzhang.markdown-all-in-one",
-		"shd101wyy.markdown-preview-enhanced",
-		"Gruntfuggly.todo-tree",
-		"redhat.vscode-yaml",
-		"PKief.material-icon-theme",
-		"mikestead.dotenv",
-		"KevinRose.vsc-python-indent",
-		"rangav.vscode-thunder-client",
-		"ms-python.black-formatter",
-		"LittleFoxTeam.vscode-python-test-adapter"
-	],
 	"remoteEnv": {
 		"LOCAL_WORKSPACE_PATH": "${localWorkspaceFolder}",
 		"PATH": "${containerWorkspaceFolder}/.venv/bin:${containerEnv:PATH}",
@@ -70,26 +29,72 @@
 		"DEMISTO_VERIFY_SSL": "${localEnv:DEMISTO_VERIFY_SSL}",
 		"DEMISTO_API_KEY": "${localEnv:DEMISTO_API_KEY}"
 	},
-	// this is commented out until VSCode will fix self signed certificate issues
-	// "features": {
-	// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/git:1": {
-	// 		"version": "os-provided"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/powershell:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/common-utils:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/node:1": {
-	// 		"version": "latest"
-	// 	}
-	// },
-	// "overrideFeatureInstallOrder": [
-	// 	"ghcr.io/devcontainers/features/common-utils:1"
-	// ],
-	"onCreateCommand": "dos2unix .devcontainer/createCommand.sh && chmod +x .devcontainer/createCommand.sh && bash .devcontainer/createCommand.sh",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.defaultProfile.linux": "zsh",
+				"terminal.integrated.profiles.linux": {
+					"zsh": {
+						"path": "/bin/zsh"
+					}
+				},
+				"cSpell.words": [
+					"demisto",
+					"xsoar",
+					"xsiam",
+					"fromversion",
+					"toversion",
+					"marketplacev",
+					"ciac",
+					"whois",
+				]
+			},
+			"extensions": [
+				"cortexxsoarext.xsoar",
+				"ms-python.python",
+				"ms-vscode.PowerShell",
+				"ms-toolsai.jupyter",
+				"timonwong.shellcheck",
+				"GitHub.vscode-pull-request-github",
+				"eamodio.gitlens",
+				"ms-azuretools.vscode-docker",
+				"ms-vscode-remote.remote-containers",
+				"streetsidesoftware.code-spell-checker",
+				"njpwerner.autodocstring",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"yzhang.markdown-all-in-one",
+				"shd101wyy.markdown-preview-enhanced",
+				"Gruntfuggly.todo-tree",
+				"redhat.vscode-yaml",
+				"PKief.material-icon-theme",
+				"mikestead.dotenv",
+				"KevinRose.vsc-python-indent",
+				"rangav.vscode-thunder-client",
+				"ms-python.black-formatter",
+				"LittleFoxTeam.vscode-python-test-adapter"
+			]
+	}
+},
+// this is commented out until VSCode will fix self signed certificate issues
+// "features": {
+// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/git:1": {
+// 		"version": "os-provided"
+// 	},
+// 	"ghcr.io/devcontainers/features/powershell:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/common-utils:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/node:1": {
+// 		"version": "latest"
+// 	}
+// },
+// "overrideFeatureInstallOrder": [
+// 	"ghcr.io/devcontainers/features/common-utils:1"
+// ],
+"onCreateCommand": "dos2unix -n .devcontainer/createCommand.sh .devcontainer/createCommand_unix.sh && chmod +x .devcontainer/createCommand_unix.sh && bash .devcontainer/createCommand_unix.sh"
 }

.github/CODEOWNERS

@@ -7,7 +7,7 @@
 /Tests/Marketplace/core_packs_mpv2_list.json @yaakovpraisler @bakatzir @GuyLibermanPA @demisto/content-leaders
 
 # Docker native image
-/Tests/native_image_config.json @GuyAfik @JudahSchwartz
+/Tests/docker_native_image_config.json @GuyAfik @JudahSchwartz
 
 # Marketplace & Upload-Flow
 /Tests/scripts/create_artifacts_graph/create_artifacts.py @ilaner
@@ -39,7 +39,7 @@
 
 # Important Integrations
 /Packs/Jira/Integrations/JiraV2/* @demisto/content-leaders
-/Packs/ServiceNow/Integrations/ServiceNowv2/* @demisto/content-leaders
+/Packs/ServiceNow/Integrations/ServiceNowv2/* @demisto/content-leaders @Shellyber
 /Packs/EWS/Integrations/EWSv2/* @demisto/content-leaders
 /Packs/ArcSightESM/Integrations/ArcSightESMv2/* @demisto/content-leaders
 /Packs/QRadar/Integrations/QRadar_v3/* @tkatzir

.github/workflows/check-devcontainer.yml

@@ -0,0 +1,21 @@
+name: Check Devcontainer
+on:
+  pull_request:
+    paths:
+      - .devcontainer/**
+  push:
+    branches:
+      - master
+    paths:
+      - .devcontainer/**
+jobs:
+  Build-Devcontainer:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Run demisto-sdk in devcontainer
+      uses: devcontainers/ci@v0.2
+      with:    
+        push: never
+        runCmd: demisto-sdk --version

.github/workflows/close_jira_issue_by_pr_merge.yml

@@ -32,11 +32,10 @@ jobs:
           PR_TITLE: ${{ github.event.pull_request.title }}
           PR_LINK: ${{ github.event.pull_request.html_url }}
           PR_BODY: ${{ github.event.pull_request.body }}
-          IS_MERGED: True
           USERNAME: ${{ secrets.LINK_PR_TO_JIRA_ISSUE_USER }}
           PASSWORD: ${{ secrets.LINK_PR_TO_JIRA_ISSUE_PASSWORD }}
           INSTANCE_URL: ${{ secrets.ENGINE_URL }}
         run: |
           echo "Checking for related Jira issues to PR: $PR_NUMBER"
           cd Utils/github_workflow_scripts/jira_integration_scripts
-          poetry run python ./link_pr_to_jira_issue.py --url "$INSTANCE_URL" --pr_num "$PR_NUMBER" --pr_link "$PR_LINK" --pr_title "$PR_TITLE" --pr_body "$PR_BODY" --is_merged $IS_MERGED --username $USERNAME --password $PASSWORD
+          poetry run python ./link_pr_to_jira_issue.py --url "$INSTANCE_URL" --pr_num "$PR_NUMBER" --pr_link "$PR_LINK" --pr_title "$PR_TITLE" --pr_body "$PR_BODY" --is_merged --username $USERNAME --password $PASSWORD

.github/workflows/link_edited_pr_to_jira_issue.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - name: Setup Poetry
         uses: Gr1N/setup-poetry@v7
       - uses: actions/cache@v2
@@ -32,12 +32,11 @@ jobs:
           PR_TITLE: ${{ github.event.pull_request.title }}
           PR_LINK: ${{ github.event.pull_request.html_url }}
           PR_BODY: ${{ github.event.pull_request.body }}
-          IS_MERGED: false
           USERNAME: ${{ secrets.LINK_PR_TO_JIRA_ISSUE_USER }}
           PASSWORD: ${{ secrets.LINK_PR_TO_JIRA_ISSUE_PASSWORD }}
           INSTANCE_URL: ${{ secrets.ENGINE_URL }}
         run: |
           echo "Checking for related Jira issues to PR: $PR_NUMBER"
           cd Utils/github_workflow_scripts/jira_integration_scripts
-          echo --pr_num $PR_NUMBER --pr_link $PR_LINK --pr_title $PR_TITLE --pr_body $PR_BODY --is_merged $IS_MERGED
-          poetry run python ./link_pr_to_jira_issue.py --url "$INSTANCE_URL" --pr_num "$PR_NUMBER" --pr_link "$PR_LINK" --pr_title "$PR_TITLE" --pr_body "$PR_BODY" --is_merged $IS_MERGED --username $USERNAME --password $PASSWORD
+          echo --pr_num $PR_NUMBER --pr_link $PR_LINK --pr_title $PR_TITLE --pr_body $PR_BODY --no-is_merged
+          poetry run python ./link_pr_to_jira_issue.py --url "$INSTANCE_URL" --pr_num "$PR_NUMBER" --pr_link "$PR_LINK" --pr_title "$PR_TITLE" --pr_body "$PR_BODY" --no-is_merged --username $USERNAME --password $PASSWORD

.gitignore

@@ -39,8 +39,7 @@ Scripts/**/__init__.py
 Integrations/**/__init__.py
 Packs/**/__init__.py
 requirements.txt
-Tests/scripts/infrastructure_tests/collect_tests.log
-Tests/scripts/collect_tests.log
+**/collect_tests.log
 
 # Environment and reports
 .demisto-sdk-conf
@@ -56,6 +55,7 @@ Packs/**/report_pytest.json
 
 # devcontainer folders
 .devcontainer/certs.crt
+.devcontainer/createCommand_unix.sh
 **/.devcontainer
 !/.devcontainer
 

.gitlab/ci/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41253
+  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41284
   artifacts:
     expire_in: 30 days
     paths:
@@ -9,16 +9,16 @@ default:
 
 stages:
   - unittests-and-validations
-  - create-instances
+  - prepare-testing-bucket
   - run-instances
-  - stage-compare
   - upload-to-marketplace
   - fan-in  # concentrate pipeline artifacts to single job before triggering child slack pipeline
 
 
 variables:
   DONT_CACHE_LAST_RESPONSE: "true"
   GCS_MARKET_BUCKET: "marketplace-dist"
+  GCS_MARKET_BUCKET_DEV: "marketplace-dist-dev"
   GCS_MARKET_V2_BUCKET: "marketplace-v2-dist"
   GCS_MARKET_XPANSE_BUCKET: "xpanse-dist"
   STORAGE_BASE_PATH: ""
@@ -42,6 +42,7 @@ variables:
   DEMISTO_CONNECTION_POOL_MAXSIZE: "180" # see this issue for more info https://github.com/demisto/etc/issues/36886
   SDK_REF: "master" # The default sdk branch to use
   OVERRIDE_ALL_PACKS: "false"
+  TEST_UPLOAD: "true"
 
 
 include:
@@ -51,4 +52,3 @@ include:
   - local: .gitlab/ci/instance-test.yml
   - local: .gitlab/ci/sdk-nightly.yml
   - local: .gitlab/ci/miscellaneous.yml
-  - local: .gitlab/ci/staging.yml