smoke - run legacy tests again, do not fail new playwright tests for a while to figure out flakiness

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -249,6 +249,7 @@ steps:
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - script: |
@@ -259,6 +260,27 @@ steps:
       yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
     timeoutInMinutes: 10
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      yarn smoketest-no-compile --legacy --tracing --build "$APP_ROOT/$APP_NAME"
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    displayName: Run smoke tests (Legacy, Electron)
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
+      yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_ROOT/$APP_NAME"
+    timeoutInMinutes: 10
+    displayName: Run smoke tests (Legacy, Remote)
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0

build/azure-pipelines/linux/product-build-linux-client.yml

@@ -271,6 +271,7 @@ steps:
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - script: |
@@ -280,6 +281,25 @@ steps:
       yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
     timeoutInMinutes: 10
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+      yarn smoketest-no-compile --legacy --tracing --build "$APP_PATH"
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    displayName: Run smoke tests (Legacy, Electron)
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
+      yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_PATH"
+    timeoutInMinutes: 10
+    displayName: Run smoke tests (Legacy, Remote)
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0

build/azure-pipelines/win32/product-build-win32.yml

@@ -231,6 +231,7 @@ steps:
       $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
       exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
@@ -242,6 +243,27 @@ steps:
       $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
       exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    timeoutInMinutes: 10
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --legacy --tracing --build "$AppRoot" }
+    displayName: Run smoke tests (Legacy, Electron)
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+      $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --legacy --tracing --remote --build "$AppRoot" }
+    displayName: Run smoke tests (Legacy, Remote)
     timeoutInMinutes: 10
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 

test/smoke/src/main.ts

@@ -377,7 +377,7 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : opts.legacy ? 'Electron (lega
 	setupSearchTests(logger);
 	setupNotebookTests(logger);
 	setupLanguagesTests(logger);
-	setupTerminalTests(logger);
+	if (!opts.legacy) { setupTerminalTests(logger); }
 	setupStatusbarTests(logger);
 	if (quality !== Quality.Dev) { setupExtensionTests(logger); }
 	setupMultirootTests(logger);