refactor(ci): Upload stage telemetry separately for each stage

scripts/get-test-pass-rate.mjs

@@ -8,64 +8,41 @@
 const BUILD_ID = process.env.BUILD_ID;
 // The token need to make the API calls.
 const ADO_API_TOKEN = process.env.ADO_API_TOKEN;
-
+// The id of the stage for which to retrieve the test pass rate information
+const STAGE_ID = process.env.STAGE_ID;
 // The workspace where the new files/folder created in this script will be stored.
-const BASE_OUTPUT_FOLDER = process.env.BASE_OUTPUT_FOLDER;
+const WORK_FOLDER = process.env.WORK_FOLDER;
+
 if (
 	BUILD_ID === undefined ||
+	STAGE_ID === undefined ||
 	ADO_API_TOKEN === undefined ||
-	BASE_OUTPUT_FOLDER === undefined
+	WORK_FOLDER === undefined
 ) {
 	throw new Error(
-		"One or more required environment variables are undefined. Please specify 'BUILD_ID', 'ADO_API_TOKEN', and 'BASE_OUTPUT_FOLDER' in order to run this script.",
+		"One or more required environment variables are undefined. Please specify 'BUILD_ID', 'STAGE_ID', 'ADO_API_TOKEN', and 'WORK_FOLDER' in order to run this script.",
 	);
 }
 console.log("BUILD_ID:", BUILD_ID);
-console.log("BASE_OUTPUT_FOLDER:", BASE_OUTPUT_FOLDER);
+console.log("STAGE_ID:", STAGE_ID);
+console.log("WORK_FOLDER:", WORK_FOLDER);
 
-// Create output folder - Note: This requires Node.js fs module
+// Create output folder
 import * as fs from "fs";
-if (!fs.existsSync(`${BASE_OUTPUT_FOLDER}/stageFiles`)) {
-	fs.mkdirSync(`${BASE_OUTPUT_FOLDER}/stageFiles`, { recursive: true });
-	console.log("Folder created");
+if (!fs.existsSync(WORK_FOLDER)) {
+	fs.mkdirSync(WORK_FOLDER, { recursive: true });
+	console.log(`Created folder '${WORK_FOLDER}'.`);
 }
-const apiUrl = `https://dev.azure.com/fluidframework/internal/_apis/build/builds/${BUILD_ID}/timeline?api-version=7.1-preview.2`;
-// Fetch data from Timeline API
-const response = await fetch(apiUrl, {
-	headers: {
-		Authorization: `Basic ${Buffer.from(":" + ADO_API_TOKEN).toString("base64")}`,
-	},
+
+// Fetch test results for the specified build + stage and save to a file
+console.log(`Fetching data for stage: ${STAGE_ID}`);
+const testResultsApiUrl = `https://vstmr.dev.azure.com/fluidframework/internal/_apis/testresults/metrics?pipelineId=${BUILD_ID}&stageName=${STAGE_ID}&api-version=7.1-preview.1`;
+const stageResponse = await fetch(testResultsApiUrl, {
+	headers: { Authorization: `Basic ${Buffer.from(":" + ADO_API_TOKEN).toString("base64")}` },
 });
-console.log(response);
-if (!response.ok) {
-	throw new Error(`Error during API call to get build timeline. Status: ${response.status}`);
+if (!stageResponse.ok) {
+	throw new Error(`Error during API call to get test results. Status: ${response.status}`);
 }
-const data = await response.json();
-console.log("Saving stage names");
-// Extract and save all stage names
-const stages = data.records
-	.filter((record) => record.type === "Stage")
-	.map((record) => record.identifier);
-for (const stage of stages) {
-	if (stage === "runAfterAll") {
-		continue;
-	}
-	console.log(`Fetching data for stage: ${stage}`);
-	// Fetch test rate data for each stage.
-	const stageApiUrl = `https://vstmr.dev.azure.com/fluidframework/internal/_apis/testresults/metrics?pipelineId=${BUILD_ID}&stageName=${stage}&api-version=7.1-preview.1`;
-	const stageResponse = await fetch(stageApiUrl, {
-		headers: {
-			Authorization: `Basic ${Buffer.from(":" + ADO_API_TOKEN).toString("base64")}`,
-		},
-	});
-	if (!stageResponse.ok) {
-		throw new Error(`Error during API call to get build metrics. Status: ${response.status}`);
-	}
 
-	const stageData = await stageResponse.json();
-	// Save the API data to a JSON file.
-	fs.writeFileSync(
-		`${BASE_OUTPUT_FOLDER}/stageFiles/${stage}.json`,
-		JSON.stringify(stageData),
-	);
-}
+const stageData = await stageResponse.json();
+fs.writeFileSync(`${WORK_FOLDER}/${STAGE_ID}.json`, JSON.stringify(stageData));

tools/pipelines/templates/build-npm-package.yml

@@ -609,28 +609,25 @@ extends:
           condition: succeededOrFailed()
           dependsOn:
             - build
-            # NOTE: This is brittle; we need to only apply these stage dependencies when the corresponding stages actually
-            # get created in the pipeline, in the include-publish-npm-package.yml file, so we want to match the compile-time
-            # conditions *and exact stage names* that exist there. At some point it might be preferable to always create the
-            # stages, control their execution with 'condition:', and update this stage to always depend on all previous
-            # stages (while still running if some of the dependencies were skipped).
-            - ${{ if eq(parameters.publish, true) }}:
-              - ${{ if eq(variables['testBuild'], true) }}:
-                - publish_npm_internal_test
-              - ${{ if eq(variables['testBuild'], false) }}:
-                - publish_npm_internal_build
-              - ${{ if and(eq(variables['testBuild'], false), eq(parameters.isReleaseGroup, true)) }}:
-                - publish_npm_internal_dev
-              - ${{ if or(eq(variables['release'], 'release'), eq(variables['release'], 'prerelease')) }}:
-                - publish_npm_public
+            # Note: the publish stages are created in include-publish-npm-package.yml. We need to match the ids exactly.
+            - publish_npm_internal_test
+            - publish_npm_internal_build
+            - publish_npm_public
+            # NOTE: This is brittle; since the publish_npm_internal_dev stage is addded to the pipeline conditionally,
+            # we create a dependency on it based on the same condition.
+            # So this needs to be kept in sync with the logic that include-publish-npm-package.yml uses to create the stage.
+            # At some point it might be preferable to always create the stage, control its execution solely with
+            # 'condition:', and update this bit to always depend on publish_npm_internal_dev, since it will always exist.
+            - ${{ if eq(parameters.isReleaseGroup, true) }}:
+              - publish_npm_internal_dev
           jobs:
             - job: upload_run_telemetry
               displayName: Upload pipeline run telemetry to Kusto
               pool: Small-1ES
               variables:
               - group: ado-feeds
               - name: pipelineTelemetryWorkdir
-                value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir
+                value: $(Pipeline.Workspace)/pipelineTelemetryWorkdir/timingOutput
                 readonly: true
               - name: absolutePathToTelemetryGenerator
                 value: $(Build.SourcesDirectory)/tools/telemetry-generator
@@ -642,21 +639,30 @@ extends:
                   officeFeedUrl: $(ado-feeds-office)
                   isCheckoutNeeded: true
               - task: Bash@3
-                displayName: Retrieve buildId results
+                displayName: Get stage timing and result data from ADO
+                env:
+                  BUILD_ID: $(Build.BuildId)
+                  ADO_API_TOKEN: $(System.AccessToken)
+                  WORK_FOLDER: $(pipelineTelemetryWorkdir)
                 inputs:
                   targetType: inline
-                  workingDirectory: $(absolutePathToTelemetryGenerator)
                   script: |
-                    echo "Creating output folder ..."
-                    mkdir -p $(pipelineTelemetryWorkdir)/timingOutput
-                    echo "Retrieving pipeline run timeline data ..."
-                    echo 'curl -u ":<REDACTED>" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline"'
-                    curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline\?api-version\=6.0-preview.1" > $(pipelineTelemetryWorkdir)/timingOutput/output.json
-                    pwd;
-                    ls -laR $(pipelineTelemetryWorkdir)/timingOutput/output.json;
-                    cat $(pipelineTelemetryWorkdir)/timingOutput/output.json;
-                    node --require @ff-internal/aria-logger bin/run --handlerModule $(absolutePathToTelemetryGenerator)/dist/handlers/stageTimingRetriever.js --dir '$(pipelineTelemetryWorkdir)/timingOutput/';
+                    echo "Creating work folder '$WORK_FOLDER'";
+                    mkdir -p $WORK_FOLDER;
+
+                    echo "Retrieving data from ADO API";
+                    echo "curl -u \":<REDACTED>\" \"https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2\""
+                    curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline?api-version=7.1-preview.2" > $WORK_FOLDER/output.json
+              - task: Bash@3
+                displayName: Submit telemetry for stage timing and result
                 env:
                   BUILD_ID: $(Build.BuildId)
-                  ADO_API_TOKEN: $(System.AccessToken)
                   PIPELINE: BuildClient
+                  WORK_FOLDER: $(pipelineTelemetryWorkdir)
+                inputs:
+                  targetType: inline
+                  workingDirectory: $(absolutePathToTelemetryGenerator)
+                  script: |
+                    echo "Listing files in '$WORK_FOLDER'"
+                    ls -laR $WORK_FOLDER;
+                    node --require @ff-internal/aria-logger bin/run --handlerModule "$(absolutePathToTelemetryGenerator)/dist/handlers/stageTimingRetriever.js" --dir "$WORK_FOLDER";

tools/pipelines/templates/include-conditionally-run-stress-tests.yml

@@ -61,24 +61,24 @@ stages:
             AFFECTED_PATHS: ${{ join(';', package.affectedPaths) }}
 
   - ${{ each package in parameters.packages }}:
-    # The ids for these stages should be kept in sync with the dependency on them in tools/pipelines/test-dds-stress.yml
-    - stage: ${{ replace(replace(package.name, '@fluidframework/', ''), '@fluid-experimental/', 'experimental_') }}_stress_tests
-      dependsOn: CheckAffectedPaths
-      displayName: Run ${{ package.name }} stress tests
-      jobs:
-        - template: include-test-real-service.yml
-          parameters:
-            # Ideally this would be a condition on the stage rather than the job, but it doesn't seem like that is supported (and ADO UI gives very little debug information
-            # as to what might be going wrong). This only impacts the "stage" view of the pipeline, in that packages with skipped tests will show up as successful stages
-            # rather than skipped stages. Clicking on a skipped stage still shows that the corresponding test job wasn't run.
-            condition: eq(stageDependencies.CheckAffectedPaths.Job.outputs['Check${{ replace(package.testFileTarName, '-', '') }}.AffectedFilesModified'],'true')
-            poolBuild: ${{ parameters.pool }}
-            loggerPackage: ''
-            artifactBuildId: ${{ parameters.artifactBuildId }}
-            testPackage: ${{ package.name }}
-            testWorkspace: ${{ parameters.testWorkspace }}
-            timeoutInMinutes: 120
-            testFileTarName: ${{ package.testFileTarName }}
-            testCommand: ${{ package.testCommand }}
-            env:
-              FUZZ_STRESS_RUN: true
+    - template: /tools/pipelines/templates/include-test-real-service.yml@self
+      parameters:
+        stageId: ${{ replace(replace(package.name, '@fluidframework/', ''), '@fluid-experimental/', 'experimental_') }}_stress_tests
+        stageDisplayName: Run ${{ package.name }} stress tests
+        stageDependencies:
+          - CheckAffectedPaths
+        pipelineIdentifierForTelemetry: 'DdsStressService'
+        # Ideally this would be a condition on the stage rather than the job, but it doesn't seem like that is supported (and ADO UI gives very little debug information
+        # as to what might be going wrong). This only impacts the "stage" view of the pipeline, in that packages with skipped tests will show up as successful stages
+        # rather than skipped stages. Clicking on a skipped stage still shows that the corresponding test job wasn't run.
+        condition: eq(stageDependencies.CheckAffectedPaths.Job.outputs['Check${{ replace(package.testFileTarName, '-', '') }}.AffectedFilesModified'],'true')
+        poolBuild: ${{ parameters.pool }}
+        loggerPackage: ''
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+        testPackage: ${{ package.name }}
+        testWorkspace: ${{ parameters.testWorkspace }}
+        timeoutInMinutes: 120
+        testFileTarName: ${{ package.testFileTarName }}
+        testCommand: ${{ package.testCommand }}
+        env:
+          FUZZ_STRESS_RUN: true