Lorrin/20260103 fix platform cd env

.github/actions/configure_azureml_agent/action.yml

@@ -70,4 +70,11 @@ runs:
       shell: bash
       run: |
         python -m pip install --upgrade pip
-        python -m pip install --upgrade -r .github/requirements/execute_job_requirements.txt
+        python -m pip install --upgrade -r .github/requirements/execute_job_requirements.txt
+
+    - name: Pre-install Azure ML CLI Extension
+      shell: bash
+      run: |
+        echo "Pre-installing Azure ML CLI extension to avoid warnings..."
+        az extension add --name ml --yes --only-show-errors 2>/dev/null || true
+        echo "Azure ML CLI extension ready."

.github/workflows/docker_taxi_cd_pipeline.yml

@@ -1,6 +1,17 @@
 name: Custom Object Detection CD Workflow
 
 on:
+  workflow_dispatch:
+    inputs:
+      exec_environment:
+        type: string
+        default: "dev"
+      model_type:
+        type: string
+        default: "docker_taxi"
+      enable_storage_public_access:
+        type: boolean
+        default: true
   push:
     branches:
       - main
@@ -22,12 +33,19 @@ on:
         description: "The type of model to run the workflow for"
         required: true
         default: "docker_taxi"
+      enable_storage_public_access:
+        type: boolean
+        description: "Temporarily enable storage public access for training"
+        required: false
+        default: true
 permissions:
   id-token: write
-  contents: read          
+  contents: read
 jobs:
   run-cd-workflow:
     uses: ./.github/workflows/platform_cd_workflow.yml
     with:
       exec_environment: ${{ inputs.exec_environment || 'dev' }}
       model_type: ${{ inputs.model_type || 'docker_taxi' }}
+      enable_storage_public_access: ${{ inputs.enable_storage_public_access != false }}
+    secrets: inherit

.github/workflows/docker_taxi_ci_pipeline.yml

@@ -11,7 +11,7 @@ on:
       - 'model/docker_taxi/**'
       - 'src/docker_taxi_src/**'
       - 'test/docker_taxi/**'
-  workflow_dispatch:
+  workflow_call:
     inputs:
       exec_environment:
         type: string
@@ -28,30 +28,20 @@ on:
         description: "Is Docker used for build validation?"
         required: true
         default: true
-  workflow_call:
-    inputs:
-      exec_environment:
-        type: string
-        description: "The environment to run the workflow in"
-        required: true
-        default: "pr"
-      model_type:
-        type: string
-        description: "The type of model to run the workflow for"
-        required: true
-        default: "docker_taxi"
-      is_docker:
+      enable_storage_public_access:
         type: boolean
-        description: "Is Docker used for build validation?"
-        required: true
+        description: "Temporarily enable storage public access for training"
+        required: false
         default: true
 permissions:
   id-token: write
-  contents: read    
+  contents: read
 jobs:
   run-ci-workflow:
     uses: ./.github/workflows/platform_ci_workflow.yml
     with:
       exec_environment: ${{ inputs.exec_environment || 'pr' }}
       model_type: ${{ inputs.model_type || 'docker_taxi' }}
       is_docker: ${{ github.event_name == 'pull_request' && true || inputs.is_docker }}
+      enable_storage_public_access: true
+    secrets: inherit

.github/workflows/london_taxi_cd_pipeline.yml

@@ -8,6 +8,9 @@ on:
       model_type:
         type: string
         default: "london_taxi"
+      enable_storage_public_access:
+        type: boolean
+        default: true
   push:
     branches:
       - main
@@ -29,6 +32,11 @@ on:
         description: "The type of model to run the workflow for"
         required: true
         default: "london_taxi"
+      enable_storage_public_access:
+        type: boolean
+        description: "Temporarily enable storage public access for training"
+        required: false
+        default: true
 permissions:
   id-token: write
   contents: read
@@ -38,3 +46,8 @@ jobs:
     with:
       exec_environment: ${{ inputs.exec_environment || 'dev' }}
       model_type: ${{ inputs.model_type || 'london_taxi' }}
+      # Converts the input parameter 'enable_storage_public_access' to a boolean value.
+      # If the input is not explicitly set to false, it defaults to true.
+      # This ensures that storage public access is enabled by default unless explicitly disabled.
+      # The double negation (!= false) handles cases where the input might be null, undefined, or any truthy value.
+      enable_storage_public_access: ${{ inputs.enable_storage_public_access != false }}

.github/workflows/london_taxi_ci_pipeline.yml

@@ -1,14 +1,6 @@
 name: London Taxi CI Workflow
 
 on:
-  workflow_dispatch:
-    inputs:
-      exec_environment:
-        type: string
-        default: "pr"
-      model_type:
-        type: string
-        default: "london_taxi"
   pull_request:
     branches:
       - main
@@ -31,6 +23,11 @@ on:
         description: "The type of model to run the workflow for"
         required: true
         default: "london_taxi"
+      enable_storage_public_access:
+        type: boolean
+        description: "Temporarily enable storage public access for training"
+        required: false
+        default: true
 permissions:
   id-token: write
   contents: read
@@ -41,4 +38,5 @@ jobs:
       exec_environment: ${{ inputs.exec_environment || 'pr' }}
       model_type: ${{ inputs.model_type || 'london_taxi' }}
       is_docker: false
+      enable_storage_public_access: true
     secrets: inherit

.github/workflows/platform_cd_workflow.yml

@@ -5,17 +5,50 @@ on:
     inputs:
       exec_environment:
         type: string
+      enable_storage_public_access:
+        type: boolean
+        default: true
+      model_type:
+        type: string
+        required: true
+  workflow_call:
+    inputs:
+      exec_environment:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
       enable_storage_public_access:
         type: boolean
         default: false
       model_type:
         type: string
+        description: "type of model to execute"
         required: true
 
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  SUBSCRIPTION_ID: ${{vars.SUBSCRIPTION_ID}}
+  RESOURCE_GROUP_NAME: ${{ vars.RESOURCE_GROUP_NAME }}
+  WORKSPACE_NAME: ${{ vars.WORKSPACE_NAME }}
+  STORAGE_ACCOUNT_NAME: ${{ vars.STORAGE_ACCT_NAME }}
+  ARM_CLIENT_ID: ${{vars.ARM_CLIENT_ID}}
+  ARM_TENANT_ID: ${{vars.ARM_TENANT_ID}}
+  BUILD_BUILDID: ${{ github.run_id }}
+  BUILD_SOURCEBRANCHNAME: ${{ github.head_ref || github.ref_name }}
+
+
+
+
+
 jobs:
   execute-training-job:
     name: Execute training job
     runs-on: ubuntu-latest
+    environment: ${{ inputs.exec_environment }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -82,6 +115,7 @@ jobs:
   deploy-online:
     name: Deploy_Online
     runs-on: ubuntu-latest
+    environment: ${{ inputs.exec_environment }}
     permissions:
       id-token: write
       contents: read
@@ -108,86 +142,6 @@ jobs:
               echo "No common directory found for ${{ inputs.model_type }}, skipping copy"
             fi
 
-      - name: Validate Storage Configuration (RBAC smoke tests)
-        run: |
-          echo "=== Validating Storage Configuration and AD-auth model download ==="
-
-          # Print storage configuration for diagnostics
-          az storage account show \
-            --name ${{ env.STORAGE_ACCOUNT_NAME }} \
-            --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
-            --subscription ${{ env.SUBSCRIPTION_ID }} \
-            --query '{name:name, defaultAction:networkRuleSet.defaultAction, allowSharedKeyAccess:allowSharedKeyAccess, publicNetworkAccess:publicNetworkAccess}' -o json
-
-          echo "\nTesting blob access with workflow identity (auth-mode login)..."
-          if ! az storage container list \
-            --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
-            --auth-mode login \
-            --subscription ${{ env.SUBSCRIPTION_ID }} \
-            --output table; then
-            echo "❌ ERROR: Workflow identity cannot access storage - this indicates RBAC or network issues"
-            exit 1
-          fi
-
-          echo "\nDeriving published model name for smoke download test..."
-          PUBLISHED_MODEL_NAME=$(python - <<PY
-          from mlops.common.naming_utils import generate_model_name
-          print(generate_model_name("${{ inputs.model_type }}"))
-          PY
-          )
-                    echo "Published model name: $PUBLISHED_MODEL_NAME"
-
-                    echo "Listing registered models to find latest version..."
-                    MODEL_LIST_JSON=$(az ml model list \
-                      --name "$PUBLISHED_MODEL_NAME" \
-                      --workspace-name ${{ env.WORKSPACE_NAME }} \
-                      --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
-                      --subscription ${{ env.SUBSCRIPTION_ID }} -o json)
-
-                    if [ -z "$MODEL_LIST_JSON" ] || [ "$MODEL_LIST_JSON" = "[]" ]; then
-                      echo "❌ ERROR: No registered model found named '$PUBLISHED_MODEL_NAME'"
-                      exit 1
-                    fi
-
-                    LATEST_VERSION=$(echo "$MODEL_LIST_JSON" | python - <<PY
-          import sys, json
-          info = json.load(sys.stdin)
-          if not info:
-              sys.exit(2)
-          print(info[0].get('version'))
-          PY
-          )
-
-                    echo "Latest model version: $LATEST_VERSION"
-
-                    echo "Attempting model download using AD auth (auth-mode login)..."
-                    if ! az ml model download \
-                      --name "$PUBLISHED_MODEL_NAME" \
-                      --version "$LATEST_VERSION" \
-                      --download-path /tmp/model-download \
-                      --auth-mode login \
-                      --workspace-name ${{ env.WORKSPACE_NAME }} \
-                      --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
-                      --subscription ${{ env.SUBSCRIPTION_ID }}; then
-                      echo "❌ ERROR: Model download via AD auth failed - this indicates missing RBAC on the model storage or workspace"
-                      exit 1
-                    fi
-
-                    echo "✅ Storage RBAC smoke tests passed (workflow identity can list blobs and download model)."
-
-                    # Test actual blob access
-                    echo "Testing blob access with workflow identity..."
-                    if ! az storage container list \
-                      --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
-                      --auth-mode login \
-                      --subscription ${{ env.SUBSCRIPTION_ID }} \
-                      --output table; then
-                      echo "❌ ERROR: Workflow identity cannot access storage - this indicates RBAC or network issues"
-                      exit 1
-                    fi
-
-                    echo "✅ Storage configuration validated successfully"
-
       - name: Provision azureml online endpoint
         uses: ./.github/actions/execute_shell_code
         with:
@@ -283,6 +237,7 @@ jobs:
   deploy-batch:
     name: Deploy_Batch
     runs-on: ubuntu-latest
+    environment: ${{ inputs.exec_environment }}
     permissions:
       id-token: write
       contents: read
@@ -361,8 +316,7 @@ jobs:
       id-token: write
       contents: read
     needs: [execute-training-job]
-    environment:
-        name: dev
+    environment: ${{ inputs.exec_environment }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/platform_ci_workflow.yml

@@ -14,7 +14,7 @@ on:
         default: false
       enable_storage_public_access:
         type: boolean
-        default: false
+        default: true
   workflow_call:
     inputs:
       exec_environment:
@@ -35,16 +35,7 @@ on:
         type: boolean
         description: "Temporarily enable storage public network access for this run"
         required: false
-        default: false
-      # arm_client_id:
-      #   required: true
-      #   type: string
-      # arm_tenant_id:
-      #   required: true
-      #   type: string
-      # subscription_id:
-      #   required: true
-      #   type: string
+        default: true
 
 permissions:
   id-token: write
@@ -65,6 +56,7 @@ jobs:
     name: Enable Storage Public Access
     if: ${{ inputs.enable_storage_public_access }}
     runs-on: ubuntu-latest
+    environment: ${{ inputs.exec_environment }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -125,6 +117,7 @@ jobs:
   execute-ml-job-pipeline:
     name: Execute ML Job Pipeline
     runs-on: ubuntu-latest
+    environment: ${{ inputs.exec_environment }}
     needs: build-validation
     steps:
       - name: Checkout
@@ -136,10 +129,6 @@ jobs:
           arm_client_id: ${{ env.ARM_CLIENT_ID }}
           arm_tenant_id: ${{ env.ARM_TENANT_ID }}
           subscription_id: ${{ env.SUBSCRIPTION_ID }}
-        # with:
-        #   arm_client_id: ${{ vars.ARM_CLIENT_ID }}
-        #   arm_tenant_id: ${{ vars.ARM_TENANT_ID }}
-        #   subscription_id: ${{ vars.SUBSCRIPTION_ID }}
 
       - name: Execute AzureML Pipeline
         uses: ./.github/actions/execute_shell_code