diff --git a/.github/actions/configure_azureml_agent/action.yml b/.github/actions/configure_azureml_agent/action.yml index 1790df53..194f726d 100644 --- a/.github/actions/configure_azureml_agent/action.yml +++ b/.github/actions/configure_azureml_agent/action.yml @@ -70,4 +70,11 @@ runs: shell: bash run: | python -m pip install --upgrade pip - python -m pip install --upgrade -r .github/requirements/execute_job_requirements.txt \ No newline at end of file + python -m pip install --upgrade -r .github/requirements/execute_job_requirements.txt + + - name: Pre-install Azure ML CLI Extension + shell: bash + run: | + echo "Pre-installing Azure ML CLI extension to avoid warnings..." + az extension add --name ml --yes --only-show-errors 2>/dev/null || true + echo "Azure ML CLI extension ready." \ No newline at end of file diff --git a/.github/workflows/docker_taxi_cd_pipeline.yml b/.github/workflows/docker_taxi_cd_pipeline.yml index ee2b0671..3463e42d 100644 --- a/.github/workflows/docker_taxi_cd_pipeline.yml +++ b/.github/workflows/docker_taxi_cd_pipeline.yml @@ -1,6 +1,17 @@ name: Custom Object Detection CD Workflow on: + workflow_dispatch: + inputs: + exec_environment: + type: string + default: "dev" + model_type: + type: string + default: "docker_taxi" + enable_storage_public_access: + type: boolean + default: true push: branches: - main @@ -22,12 +33,19 @@ on: description: "The type of model to run the workflow for" required: true default: "docker_taxi" + enable_storage_public_access: + type: boolean + description: "Temporarily enable storage public access for training" + required: false + default: true permissions: id-token: write - contents: read + contents: read jobs: run-cd-workflow: uses: ./.github/workflows/platform_cd_workflow.yml with: exec_environment: ${{ inputs.exec_environment || 'dev' }} model_type: ${{ inputs.model_type || 'docker_taxi' }} + enable_storage_public_access: ${{ inputs.enable_storage_public_access != false }} + secrets: inherit diff --git a/.github/workflows/docker_taxi_ci_pipeline.yml b/.github/workflows/docker_taxi_ci_pipeline.yml index 30cb388a..ab8d68f3 100644 --- a/.github/workflows/docker_taxi_ci_pipeline.yml +++ b/.github/workflows/docker_taxi_ci_pipeline.yml @@ -11,7 +11,7 @@ on: - 'model/docker_taxi/**' - 'src/docker_taxi_src/**' - 'test/docker_taxi/**' - workflow_dispatch: + workflow_call: inputs: exec_environment: type: string @@ -28,26 +28,14 @@ on: description: "Is Docker used for build validation?" required: true default: true - workflow_call: - inputs: - exec_environment: - type: string - description: "The environment to run the workflow in" - required: true - default: "pr" - model_type: - type: string - description: "The type of model to run the workflow for" - required: true - default: "docker_taxi" - is_docker: + enable_storage_public_access: type: boolean - description: "Is Docker used for build validation?" - required: true + description: "Temporarily enable storage public access for training" + required: false default: true permissions: id-token: write - contents: read + contents: read jobs: run-ci-workflow: uses: ./.github/workflows/platform_ci_workflow.yml @@ -55,3 +43,5 @@ jobs: exec_environment: ${{ inputs.exec_environment || 'pr' }} model_type: ${{ inputs.model_type || 'docker_taxi' }} is_docker: ${{ github.event_name == 'pull_request' && true || inputs.is_docker }} + enable_storage_public_access: true + secrets: inherit diff --git a/.github/workflows/london_taxi_cd_pipeline.yml b/.github/workflows/london_taxi_cd_pipeline.yml index 0d8c6ffd..ed22adbf 100644 --- a/.github/workflows/london_taxi_cd_pipeline.yml +++ b/.github/workflows/london_taxi_cd_pipeline.yml @@ -8,6 +8,9 @@ on: model_type: type: string default: "london_taxi" + enable_storage_public_access: + type: boolean + default: true push: branches: - main @@ -29,6 +32,11 @@ on: description: "The type of model to run the workflow for" required: true default: "london_taxi" + enable_storage_public_access: + type: boolean + description: "Temporarily enable storage public access for training" + required: false + default: true permissions: id-token: write contents: read @@ -38,3 +46,8 @@ jobs: with: exec_environment: ${{ inputs.exec_environment || 'dev' }} model_type: ${{ inputs.model_type || 'london_taxi' }} + # Converts the input parameter 'enable_storage_public_access' to a boolean value. + # If the input is not explicitly set to false, it defaults to true. + # This ensures that storage public access is enabled by default unless explicitly disabled. + # The double negation (!= false) handles cases where the input might be null, undefined, or any truthy value. + enable_storage_public_access: ${{ inputs.enable_storage_public_access != false }} diff --git a/.github/workflows/london_taxi_ci_pipeline.yml b/.github/workflows/london_taxi_ci_pipeline.yml index 3b76972a..bea07470 100644 --- a/.github/workflows/london_taxi_ci_pipeline.yml +++ b/.github/workflows/london_taxi_ci_pipeline.yml @@ -1,14 +1,6 @@ name: London Taxi CI Workflow on: - workflow_dispatch: - inputs: - exec_environment: - type: string - default: "pr" - model_type: - type: string - default: "london_taxi" pull_request: branches: - main @@ -31,6 +23,11 @@ on: description: "The type of model to run the workflow for" required: true default: "london_taxi" + enable_storage_public_access: + type: boolean + description: "Temporarily enable storage public access for training" + required: false + default: true permissions: id-token: write contents: read @@ -41,4 +38,5 @@ jobs: exec_environment: ${{ inputs.exec_environment || 'pr' }} model_type: ${{ inputs.model_type || 'london_taxi' }} is_docker: false + enable_storage_public_access: true secrets: inherit \ No newline at end of file diff --git a/.github/workflows/platform_cd_workflow.yml b/.github/workflows/platform_cd_workflow.yml index 47f733be..0b0313ef 100644 --- a/.github/workflows/platform_cd_workflow.yml +++ b/.github/workflows/platform_cd_workflow.yml @@ -5,17 +5,50 @@ on: inputs: exec_environment: type: string + enable_storage_public_access: + type: boolean + default: true + model_type: + type: string + required: true + workflow_call: + inputs: + exec_environment: + type: string + description: "Execution Environment" + required: true + default: "dev" enable_storage_public_access: type: boolean default: false model_type: type: string + description: "type of model to execute" required: true +permissions: + id-token: write + contents: read + +env: + SUBSCRIPTION_ID: ${{vars.SUBSCRIPTION_ID}} + RESOURCE_GROUP_NAME: ${{ vars.RESOURCE_GROUP_NAME }} + WORKSPACE_NAME: ${{ vars.WORKSPACE_NAME }} + STORAGE_ACCOUNT_NAME: ${{ vars.STORAGE_ACCT_NAME }} + ARM_CLIENT_ID: ${{vars.ARM_CLIENT_ID}} + ARM_TENANT_ID: ${{vars.ARM_TENANT_ID}} + BUILD_BUILDID: ${{ github.run_id }} + BUILD_SOURCEBRANCHNAME: ${{ github.head_ref || github.ref_name }} + + + + + jobs: execute-training-job: name: Execute training job runs-on: ubuntu-latest + environment: ${{ inputs.exec_environment }} steps: - name: Checkout uses: actions/checkout@v4 @@ -82,6 +115,7 @@ jobs: deploy-online: name: Deploy_Online runs-on: ubuntu-latest + environment: ${{ inputs.exec_environment }} permissions: id-token: write contents: read @@ -108,86 +142,6 @@ jobs: echo "No common directory found for ${{ inputs.model_type }}, skipping copy" fi - - name: Validate Storage Configuration (RBAC smoke tests) - run: | - echo "=== Validating Storage Configuration and AD-auth model download ===" - - # Print storage configuration for diagnostics - az storage account show \ - --name ${{ env.STORAGE_ACCOUNT_NAME }} \ - --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ - --subscription ${{ env.SUBSCRIPTION_ID }} \ - --query '{name:name, defaultAction:networkRuleSet.defaultAction, allowSharedKeyAccess:allowSharedKeyAccess, publicNetworkAccess:publicNetworkAccess}' -o json - - echo "\nTesting blob access with workflow identity (auth-mode login)..." - if ! az storage container list \ - --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \ - --auth-mode login \ - --subscription ${{ env.SUBSCRIPTION_ID }} \ - --output table; then - echo "❌ ERROR: Workflow identity cannot access storage - this indicates RBAC or network issues" - exit 1 - fi - - echo "\nDeriving published model name for smoke download test..." - PUBLISHED_MODEL_NAME=$(python - < Dict: available = ', '.join(sorted(self.pipeline_configs.keys())) raise KeyError( - f"Pipeline config '{pipelineconfig_name}' not found in {self.config_path}. ``pipeline_configs`` keys: {available}" + f"Pipeline config '{pipelineconfig_name}' not found in {self.config_path}. " + f"``pipeline_configs`` keys: {available}" ) def get_deployment_config(self, deployment_name: str) -> Dict: @@ -143,7 +144,8 @@ def get_deployment_config(self, deployment_name: str) -> Dict: available = ', '.join(sorted(self.deployment_configs.keys())) raise KeyError( - f"Deployment config '{deploymentconfig_name}' not found in {self.config_path}. ``deployment_configs`` keys: {available}" + f"Deployment config '{deploymentconfig_name}' not found in {self.config_path}. " + f"``deployment_configs`` keys: {available}" )