diff --git a/src/testRunner/externalCompileRunner.ts b/src/testRunner/externalCompileRunner.ts
index 898dccf54ea..75157d4dab4 100644
--- a/src/testRunner/externalCompileRunner.ts
+++ b/src/testRunner/externalCompileRunner.ts
@@ -138,7 +138,8 @@ function compareErrorStrings(a: string[], b: string[]) {
     return ts.comparePathsCaseSensitive(errorFileA, errorFileB) ||
         ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) ||
         ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) ||
-        ts.compareStringsCaseSensitive(remainderA, remainderB);
+        ts.compareStringsCaseSensitive(remainderA, remainderB) ||
+        ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n"));
 }
 
 class DefinitelyTypedRunner extends ExternalCompileRunnerBase {