move integration tests to step before release, improve naming of workflows

Author: emileten

.github/workflows/build.yaml

@@ -12,17 +12,30 @@ on:
         required: false
       DS_RELEASE_BOT_PRIVATE_KEY:
         required: false
-
+      AWS_DEFAULT_REGION_DEPLOY:
+        required: false
+      AWS_ACCESS_KEY_ID_DEPLOY:
+        required: false
+      AWS_SECRET_ACCESS_KEY_DEPLOY:
+        required: false
+      AWS_ACCOUNT_ID:
+        required: false
 jobs:
   build_and_package:
     name: Build and package
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }}
+      AWS_DEFAULT_ACCOUNT: ${{ secrets.AWS_ACCOUNT_ID }}
     steps:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
 
       - name: Install Dependencies
@@ -65,8 +78,86 @@ jobs:
           app_id: ${{ secrets.DS_RELEASE_BOT_ID }}
           private_key: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }}
 
-      - name: Maybe Release 🚀
+      - name: Check release
+        id: check_release
         if: ${{ inputs.release }}
+        run: |
+          SHOULD_RELEASE=false
+          npm run semantic-release --dry-run > check_release_output.txt
+          if grep -q "Published release" check_release_output.txt; then
+            echo "SHOULD_RELEASE=true" >> $GITHUB_OUTPUT
+          else
+            echo "SHOULD_RELEASE=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install deployment environment
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
+        id: install_deploy_env
+        run: |
+          # install deployment environment with eoapi-cdk from build
+          python -m venv .deployment_venv
+          source .deployment_venv/bin/activate
+          pip install dist/python/*.gz
+          cd integration_tests/cdk
+          pip install -r requirements.txt
+          npm install   
+          deactivate
+          cd -
+
+
+      - name: Deploy test stack
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
+        id: deploy_step
+        run: |
+          source .deployment_venv/bin/activate
+          
+          # synthesize the stack
+          cd integration_tests/cdk
+          npx cdk synth --debug --all --require-approval never
+
+          # deploy the stack and grab URLs for testing
+          npx cdk deploy --ci --all --require-approval never
+          echo "ingestor_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'stacingestor')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          echo "stac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'pgstacapi')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          echo "titiler_pgstac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'titilerpgstac')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          deactivate
+          cd -
+          
+      - name: Run integration tests
+        id: run_tests
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
+        env:
+          ingestor_url: ${{ steps.deploy_step.outputs.ingestor_url }}
+          stac_api_url: ${{ steps.deploy_step.outputs.stac_api_url }}
+          titiler_pgstac_api_url: ${{ steps.deploy_step.outputs.titiler_pgstac_api_url }}        
+        run: |
+          cd integration_tests/tests
+          python -m venv .tests_venv
+          source .tests_venv/bin/activate
+          pip install -e .
+          pytest eoapi_tests
+          deactivate
+          cd -
+
+      - name: Tear down any infrastructure
+        if: always()
+        run: |
+          cd integration_tests/cdk
+          # run this only if we find a 'cdk.out' directory, which means there might be things to tear down
+          if [ -d "cdk.out" ]; then
+            cd -
+            source .deployment_venv/bin/activate
+            cd integration_tests/cdk
+            # see https://github.com/aws/aws-cdk/issues/24946
+            rm -f cdk.out/synth.lock
+            npx cdk destroy --ci --all --force
+          fi
+  
+        
+      # run if the previous step set SHOULD_RELEASE to true
+      - name: Maybe Release 🚀
+        # only run if the previous step set SHOULD_RELEASE to true
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
         run: |
           npm run semantic-release
         env:

.github/workflows/build_and_release.yaml

@@ -0,0 +1,17 @@
+name: Build & try to release
+
+on:
+  push:
+
+jobs:
+  package:
+    uses: ./.github/workflows/build.yaml
+    with:
+      release: true
+    secrets:
+      DS_RELEASE_BOT_ID: ${{ secrets.DS_RELEASE_BOT_ID }}
+      DS_RELEASE_BOT_PRIVATE_KEY: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }}
+      AWS_DEFAULT_REGION_DEPLOY: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
+      AWS_ACCESS_KEY_ID_DEPLOY: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
+      AWS_SECRET_ACCESS_KEY_DEPLOY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }}
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}

.github/workflows/distribute.yaml

@@ -9,10 +9,6 @@ jobs:
   package:
     uses: ./.github/workflows/build.yaml
 
-  integration-test:
-    uses: ./.github/workflows/integration-test.yaml
-    needs: package
-
   distribute-python:
     runs-on: ubuntu-latest
     needs: package

.github/workflows/integration-test.yaml

@@ -4,12 +4,17 @@
 import yaml
 from aws_cdk import aws_ec2
 from pydantic_core.core_schema import FieldValidationInfo
-from pydantic_settings import BaseSettings
-
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 class AppConfig(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env"
+    )
+    aws_default_account: str = pydantic.Field(
+        description="AWS account ID"
+    )
     project_id: str = pydantic.Field(
-        description="Project ID", default="eoapi-template-demo"
+        description="Project ID", default="eoapi-cdk-integration"
     )
     stage: str = pydantic.Field(description="Stage of deployment", default="test")
     # because of its validator, `tags` should always come after `project_id` and `stage`

.github/workflows/test.yaml

@@ -232,7 +232,7 @@ def __init__(
         # must be already set up, or set up after this deployment.
         if not app_config.data_access_role_arn:
             data_access_role = self._grant_assume_role_with_principal_pattern(
-                data_access_role, stac_ingestor.handler_role.role_name
+                data_access_role, stac_ingestor.handler_role.role_name, app_config.aws_default_account
             )
 
     def _create_data_access_role(self) -> aws_iam.Role:
@@ -261,7 +261,7 @@ def _grant_assume_role_with_principal_pattern(
         self,
         role_to_assume: aws_iam.Role,
         principal_pattern: str,
-        account_id: str = boto3.client("sts").get_caller_identity().get("Account"),
+        account_id: str,
     ) -> aws_iam.Role:
         """
         Grants assume role permissions to the role of the given

integration_tests/cdk/config.py

@@ -2,7 +2,7 @@
     "name": "eoapi-template",
     "version": "0.1.0",
     "dependencies": {
-      "aws-cdk": "^2.81.0"
+      "aws-cdk": "^2.99.1"
     }
   }
 

integration_tests/cdk/eoapi_template/pgStacInfra.py

@@ -1,6 +1,6 @@
-aws-cdk-lib>=2.75.0
-aws_cdk.aws_cognito_identitypool_alpha>=2.75.0a0
-aws-cdk.aws-apigatewayv2-alpha==2.95.1a0
+aws-cdk-lib>=2.99.1
+aws_cdk.aws_cognito_identitypool_alpha>=2.99.0a0
+aws-cdk.aws-apigatewayv2-alpha>=2.99.0a0
 constructs>=10.0.0,<11.0.0
 pydantic==2.0.2
 pydantic-settings==2.0.1

integration_tests/cdk/package-lock.json

@@ -58,7 +58,7 @@ def send(
     headers = {"content-type": "", "content-length": str(len(json_responseBody))}
 
     try:
-        response = httpx.put(responseUrl, data=json_responseBody, headers=headers)
+        response = httpx.put(responseUrl, data=json_responseBody, headers=headers, timeout=30)
         print("Status code: " + response.status_code)
     except Exception as e:
         print("send(..) failed executing httpx.put(..): " + str(e))

integration_tests/cdk/package.json

@@ -73,8 +73,7 @@ export class PgStacDatabase extends Construct {
       // overwrites defaults with user-provided configurable properties
       ...props.bootstrapperLambdaFunctionOptions,
       // Non configurable properties that are going to be overwritten even if provided by the user
-      vpc: hasVpc(this.db) ? this.db.vpc : props.vpc,
-      allowPublicSubnet: true
+      vpc: hasVpc(this.db) ? this.db.vpc : props.vpc
     });
 
     this.pgstacSecret = new secretsmanager.Secret(this, "bootstrappersecret", {