Add Azure pipeline to run sanity tests (#287062)

* Add Azure pipeline to run sanity tests

* Refactor into job template, add tests logs upload.

* Remove unused packages from dependencies.

build/azure-pipelines/common/sanity-tests.yml

@@ -0,0 +1,61 @@
+parameters:
+  - name: commit
+    type: string
+  - name: quality
+    type: string
+  - name: poolName
+    type: string
+  - name: os
+    type: string
+
+jobs:
+  - job: ${{ parameters.os }}
+    displayName: ${{ parameters.os }} Sanity Tests
+    pool:
+      name: ${{ parameters.poolName }}
+      os: ${{ parameters.os }}
+    timeoutInMinutes: 30
+    variables:
+      SANITY_TEST_LOGS: $(Build.SourcesDirectory)/.build/sanity-test-logs
+    templateContext:
+      outputs:
+        - output: pipelineArtifact
+          targetPath: $(SANITY_TEST_LOGS)
+          artifactName: sanity-test-logs-${{ lower(parameters.os) }}-$(System.JobAttempt)
+          displayName: Publish Sanity Test Logs
+          sbomEnabled: false
+          isProduction: false
+          condition: succeededOrFailed()
+    steps:
+      - checkout: self
+
+      - task: NodeTool@0
+        inputs:
+          versionSource: fromFile
+          versionFilePath: .nvmrc
+        displayName: Install Node.js
+
+      - ${{ if eq(parameters.os, 'windows') }}:
+        - script: |
+            mkdir "$(SANITY_TEST_LOGS)"
+          displayName: Create Logs Directory
+
+      - ${{ else }}:
+        - script: |
+            mkdir -p "$(SANITY_TEST_LOGS)"
+          displayName: Create Logs Directory
+
+      - script: npm install
+        displayName: Install Dependencies
+        workingDirectory: $(Build.SourcesDirectory)/test/sanity
+
+      - script: npm run sanity-test -- --commit ${{ parameters.commit }} --quality ${{ parameters.quality }} --verbose --test-results $(SANITY_TEST_LOGS)/sanity-test.xml
+        displayName: Run Sanity Tests
+
+      - task: PublishTestResults@2
+        inputs:
+          testResultsFormat: JUnit
+          testResultsFiles: $(SANITY_TEST_LOGS)/sanity-test.xml
+          testRunTitle: ${{ parameters.os }} Sanity Tests
+        condition: succeededOrFailed()
+        displayName: Publish Test Results

build/azure-pipelines/product-sanity-tests.yml

@@ -0,0 +1,71 @@
+pr: none
+
+trigger: none
+
+parameters:
+  - name: commit
+    displayName: Commit
+    type: string
+  - name: quality
+    displayName: Quality
+    type: string
+    default: insider
+    values:
+      - exploration
+      - insider
+      - stable
+
+variables:
+  - name: skipComponentGovernanceDetection
+    value: true
+  - name: Codeql.SkipTaskAutoInjection
+    value: true
+
+name: "$(Date:yyyyMMdd).$(Rev:r) (${{ parameters.quality }})"
+
+resources:
+  repositories:
+    - repository: 1ESPipelines
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    sdl:
+      tsa:
+        enabled: false
+      codeql:
+        compiled:
+          enabled: false
+          justificationForDisabling: "Sanity tests only, no code compilation"
+      credscan:
+        suppressionsFile: $(Build.SourcesDirectory)/build/azure-pipelines/config/CredScanSuppressions.json
+      eslint:
+        enabled: false
+      sourceAnalysisPool: 1es-windows-2022-x64
+      createAdoIssuesForJustificationsForDisablement: false
+    stages:
+      - stage: SanityTests
+        jobs:
+          - template: build/azure-pipelines/common/sanity-tests.yml@self
+            parameters:
+              commit: ${{ parameters.commit }}
+              quality: ${{ parameters.quality }}
+              poolName: 1es-windows-2022-x64
+              os: windows
+
+          - template: build/azure-pipelines/common/sanity-tests.yml@self
+            parameters:
+              commit: ${{ parameters.commit }}
+              quality: ${{ parameters.quality }}
+              poolName: 1es-ubuntu-22.04-x64
+              os: linux
+
+          - template: build/azure-pipelines/common/sanity-tests.yml@self
+            parameters:
+              commit: ${{ parameters.commit }}
+              quality: ${{ parameters.quality }}
+              poolName: AcesShared
+              os: macOS

test/sanity/package.json

@@ -9,6 +9,7 @@
     "start": "node ./out/index.js"
   },
   "dependencies": {
+    "mocha-junit-reporter": "^2.2.1",
     "node-fetch": "^3.3.2",
     "playwright": "^1.57.0"
   },

test/sanity/src/context.ts

@@ -34,6 +34,7 @@ export class TestContext {
 
 	private readonly tempDirs = new Set<string>();
 	private readonly logFile: string;
+	private _currentTest?: Mocha.Test & { consoleOutputs?: string[] };
 
 	public constructor(
 		public readonly quality: 'stable' | 'insider' | 'exploration',
@@ -47,6 +48,14 @@ export class TestContext {
 		console.log(`Log file: ${this.logFile}`);
 	}
 
+	/**
+	 * Sets the current test for log capturing.
+	 */
+	public set currentTest(test: Mocha.Test) {
+		this._currentTest = test;
+		this._currentTest.consoleOutputs ||= [];
+	}
+
 	/**
 	 * Returns the current platform in the format <platform>-<arch>.
 	 */
@@ -58,10 +67,11 @@ export class TestContext {
 	 * Logs a message with a timestamp.
 	 */
 	public log(message: string) {
-		const line = `[${new Date().toISOString()}] ${message}\n`;
-		fs.appendFileSync(this.logFile, line);
+		const line = `[${new Date().toISOString()}] ${message}`;
+		fs.appendFileSync(this.logFile, line + '\n');
+		this._currentTest?.consoleOutputs?.push(line);
 		if (this.verbose) {
-			console.log(line.trimEnd());
+			console.log(line);
 		}
 	}
 
@@ -69,9 +79,10 @@ export class TestContext {
 	 * Logs an error message and throws an Error.
 	 */
 	public error(message: string): never {
-		const line = `[${new Date().toISOString()}] ERROR: ${message}\n`;
-		fs.appendFileSync(this.logFile, line);
-		console.error(line.trimEnd());
+		const line = `[${new Date().toISOString()}] ERROR: ${message}`;
+		fs.appendFileSync(this.logFile, line + '\n');
+		this._currentTest?.consoleOutputs?.push(line);
+		console.error(line);
 		throw new Error(message);
 	}
 

test/sanity/src/index.ts

@@ -7,9 +7,9 @@ import minimist from 'minimist';
 import Mocha, { MochaOptions } from 'mocha';
 
 const options = minimist(process.argv.slice(2), {
-	string: ['fgrep', 'grep'],
+	string: ['fgrep', 'grep', 'test-results'],
 	boolean: ['help'],
-	alias: { fgrep: 'f', grep: 'g', help: 'h' },
+	alias: { fgrep: 'f', grep: 'g', help: 'h', 'test-results': 't' },
 });
 
 if (options.help) {
@@ -21,17 +21,21 @@ if (options.help) {
 	console.info('  --no-signing-check              Skip Authenticode and codesign signature checks');
 	console.info('  --grep, -g <pattern>            Only run tests matching the given <pattern>');
 	console.info('  --fgrep, -f <string>            Only run tests containing the given <string>');
+	console.info('  --test-results, -t <path>       Output test results in JUnit format to the specified path');
 	console.info('  --verbose, -v                   Enable verbose logging');
 	console.info('  --help, -h                      Show this help message');
 	process.exit(0);
 }
 
+const testResults = options['test-results'];
 const mochaOptions: MochaOptions = {
 	color: true,
 	timeout: 5 * 60 * 1000,
 	slow: 3 * 60 * 1000,
 	grep: options.grep,
 	fgrep: options.fgrep,
+	reporter: testResults ? 'mocha-junit-reporter' : undefined,
+	reporterOptions: testResults ? { mochaFile: testResults, outputs: true } : undefined,
 };
 
 const mocha = new Mocha(mochaOptions);

test/sanity/src/main.ts

@@ -28,7 +28,8 @@ if (!options.quality) {
 const context = new TestContext(options.quality, options.commit, options.verbose, !options['signing-check']);
 
 describe('VS Code Sanity Tests', () => {
-	beforeEach(() => {
+	beforeEach(function () {
+		context.currentTest = this.currentTest!;
 		const cwd = context.createTempDir();
 		process.chdir(cwd);
 		context.log(`Changed working directory to: ${cwd}`);