Fix: verification of incremental correctness that was not working because of using wrote writeFile (#48751)

* Use fixed time for vfs so baselining is consistent

* Baseline buildinfos

* Write new file text in baseline even if the file wasnt read on the shadow

* Remove unnecessary debugger statement

* Make sure that incremental correctness is checked with correct writeFile so we know buildInfo was written
Also baseline these so its easy to verify the changes

* More baselines for the tsbuildinfo

* Renames and test fixes after dts Signature change merge

* COmment

src/testRunner/unittests/tsbuild/helpers.ts

@@ -209,7 +209,7 @@ interface Symbol {
         affectedFilesPendingEmit?: readonly ReadableProgramBuilderInfoFilePendingEmit[];
     }
     type ReadableBuildInfo = Omit<BuildInfo, "program"> & { program: ReadableProgramBuildInfo | undefined; size: number; };
-    function generateBuildInfoProgramBaseline(sys: System, originalWriteFile: System["writeFile"], buildInfoPath: string, buildInfo: BuildInfo) {
+    function generateBuildInfoProgramBaseline(sys: System, buildInfoPath: string, buildInfo: BuildInfo) {
         const fileInfos: ReadableProgramBuildInfo["fileInfos"] = {};
         buildInfo.program?.fileInfos.forEach((fileInfo, index) => fileInfos[toFileName(index + 1 as ProgramBuildInfoFileId)] = toBuilderStateFileInfo(fileInfo));
         const fileNamesList = buildInfo.program?.fileIdsList?.map(fileIdsListId => fileIdsListId.map(toFileName));
@@ -240,7 +240,7 @@ interface Symbol {
             size: getBuildInfoText({ ...buildInfo, version }).length,
         };
         // For now its just JSON.stringify
-        originalWriteFile.call(sys, `${buildInfoPath}.readable.baseline.txt`, JSON.stringify(result, /*replacer*/ undefined, 2));
+        sys.writeFile(`${buildInfoPath}.readable.baseline.txt`, JSON.stringify(result, /*replacer*/ undefined, 2));
 
         function toFileName(fileId: ProgramBuildInfoFileId) {
             return buildInfo.program!.fileNames[fileId - 1];
@@ -266,16 +266,15 @@ interface Symbol {
 
     export function baselineBuildInfo(
         options: CompilerOptions,
-        sys: System & { writtenFiles: ReadonlyCollection<Path>; },
+        sys: TscCompileSystem | tscWatch.WatchedSystem,
         originalReadCall?: System["readFile"],
-        originalWriteFile?: System["writeFile"],
     ) {
         const buildInfoPath = getTsBuildInfoEmitOutputFilePath(options);
-        if (!buildInfoPath || !sys.writtenFiles.has(toPathWithSystem(sys, buildInfoPath))) return;
+        if (!buildInfoPath || !sys.writtenFiles!.has(toPathWithSystem(sys, buildInfoPath))) return;
         if (!sys.fileExists(buildInfoPath)) return;
 
         const buildInfo = getBuildInfo((originalReadCall || sys.readFile).call(sys, buildInfoPath, "utf8")!);
-        generateBuildInfoProgramBaseline(sys, originalWriteFile || sys.writeFile, buildInfoPath, buildInfo);
+        generateBuildInfoProgramBaseline(sys, buildInfoPath, buildInfo);
 
         if (!outFile(options)) return;
         const { jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
@@ -288,134 +287,173 @@ interface Symbol {
         generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.dts, declarationFilePath);
         baselineRecorder.Close();
         const text = baselineRecorder.lines.join("\r\n");
-        (originalWriteFile || sys.writeFile).call(sys, `${buildInfoPath}.baseline.txt`, text);
+        sys.writeFile(`${buildInfoPath}.baseline.txt`, text);
     }
-
-    interface VerifyTscEditCorrectnessInput {
+    interface VerifyTscEditDiscrepanciesInput {
+        index: number;
         scenario: TestTscCompile["scenario"];
+        subScenario: TestTscCompile["subScenario"];
+        baselines: string[] | undefined;
         commandLineArgs: TestTscCompile["commandLineArgs"];
         modifyFs: TestTscCompile["modifyFs"];
         editFs: TestTscEdit["modifyFs"];
         baseFs: vfs.FileSystem;
         newSys: TscCompileSystem;
-        cleanBuildDiscrepancies: TestTscEdit["cleanBuildDiscrepancies"];
+        discrepancyExplanation: TestTscEdit["discrepancyExplanation"];
     }
-    /** Verify that emit is same as clean build vs building after edit */
-    function verifyTscEditCorrectness(input: () => VerifyTscEditCorrectnessInput, index: number, subScenario: TestTscCompile["subScenario"]) {
-        it(`Verify emit output file text is same when built clean for incremental edit scenario at:: ${index} ${subScenario}`, () => {
-            const {
-                scenario, commandLineArgs, cleanBuildDiscrepancies,
-                modifyFs, editFs,
-                baseFs, newSys
-            } = input();
-            const sys = testTscCompile({
-                scenario,
-                subScenario,
-                fs: () => baseFs.makeReadonly(),
-                commandLineArgs,
-                modifyFs: fs => {
-                    if (modifyFs) modifyFs(fs);
-                    editFs(fs);
-                },
-                disableUseFileVersionAsSignature: true,
-            });
-            const discrepancies = cleanBuildDiscrepancies?.();
-            for (const outputFile of arrayFrom(sys.writtenFiles.keys())) {
-                const cleanBuildText = sys.readFile(outputFile);
-                const incrementalBuildText = newSys.readFile(outputFile);
-                const descrepancyInClean = discrepancies?.get(outputFile);
-                if (isBuildInfoFile(outputFile)) {
-                    // Check only presence and absence and not text as we will do that for readable baseline
-                    assert.isTrue(sys.fileExists(`${outputFile}.readable.baseline.txt`), `Readable baseline should be present in clean build:: File:: ${outputFile}`);
-                    assert.isTrue(newSys.fileExists(`${outputFile}.readable.baseline.txt`), `Readable baseline should be present in incremental build:: File:: ${outputFile}`);
-                    if (descrepancyInClean === undefined) {
-                        verifyPresenceAbsence(incrementalBuildText, cleanBuildText, `Incremental and clean tsbuildinfo file presence should match:: File:: ${outputFile}`);
-                    }
-                    else {
-                        verifyTextEqual(incrementalBuildText, cleanBuildText, descrepancyInClean, `File: ${outputFile}`);
-                    }
-                }
-                else if (!fileExtensionIs(outputFile, ".tsbuildinfo.readable.baseline.txt")) {
-                    verifyTextEqual(incrementalBuildText, cleanBuildText, descrepancyInClean, `File: ${outputFile}`);
-                }
-                else if (incrementalBuildText !== cleanBuildText) {
-                    // Verify build info without affectedFilesPendingEmit
-                    const { buildInfo: incrementalBuildInfo, readableBuildInfo: incrementalReadableBuildInfo } = getBuildInfoForIncrementalCorrectnessCheck(incrementalBuildText);
-                    const { buildInfo: cleanBuildInfo, readableBuildInfo: cleanReadableBuildInfo } = getBuildInfoForIncrementalCorrectnessCheck(cleanBuildText);
-                    verifyTextEqual(incrementalBuildInfo, cleanBuildInfo, descrepancyInClean, `TsBuild info text without affectedFilesPendingEmit ${subScenario}:: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
-                    if (descrepancyInClean === undefined) {
-                        // Verify file info sigantures
-                        verifyMapLike(
-                            incrementalReadableBuildInfo?.program?.fileInfos,
-                            cleanReadableBuildInfo?.program?.fileInfos,
-                            (key, incrementalFileInfo, cleanFileInfo) => {
-                                if (incrementalFileInfo.signature !== cleanFileInfo.signature && incrementalFileInfo.signature !== incrementalFileInfo.version) {
-                                    assert.fail(`Incremental signature should either be dts signature or file version for File:: ${key}:: Incremental:: ${JSON.stringify(incrementalFileInfo)}, Clean:: ${JSON.stringify(cleanFileInfo)}}`);
-                                }
-                            },
-                            `FileInfos:: File:: ${outputFile}`
-                        );
-                        // Verify exportedModulesMap
-                        verifyMapLike(
-                            incrementalReadableBuildInfo?.program?.exportedModulesMap,
-                            cleanReadableBuildInfo?.program?.exportedModulesMap,
-                            (key, incrementalReferenceSet, cleanReferenceSet) => {
-                                if (!arrayIsEqualTo(incrementalReferenceSet, cleanReferenceSet) && !arrayIsEqualTo(incrementalReferenceSet, incrementalReadableBuildInfo!.program!.referencedMap![key])) {
-                                    assert.fail(`Incremental Reference set should either be from dts or files reference map for File:: ${key}:: Incremental:: ${JSON.stringify(incrementalReferenceSet)}, Clean:: ${JSON.stringify(cleanReferenceSet)}, referenceMap:: ${JSON.stringify(incrementalReadableBuildInfo!.program!.referencedMap![key])}}`);
-                                }
-                            },
-                            `exportedModulesMap:: File:: ${outputFile}`
-                        );
-                        // Verify that incrementally pending affected file emit are in clean build since clean build can contain more files compared to incremental depending of noEmitOnError option
-                        if (incrementalReadableBuildInfo?.program?.affectedFilesPendingEmit) {
-                            assert.isDefined(cleanReadableBuildInfo?.program?.affectedFilesPendingEmit, `Incremental build contains affectedFilesPendingEmit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
-                            let expectedIndex = 0;
-                            incrementalReadableBuildInfo.program.affectedFilesPendingEmit.forEach(([actualFile]) => {
-                                expectedIndex = findIndex(cleanReadableBuildInfo!.program!.affectedFilesPendingEmit!, ([expectedFile]) => actualFile === expectedFile, expectedIndex);
-                                assert.notEqual(expectedIndex, -1, `Incremental build contains ${actualFile} file as pending emit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
-                                expectedIndex++;
-                            });
-                        }
-                    }
-                }
-            }
-
-            function verifyTextEqual(incrementalText: string | undefined, cleanText: string | undefined, descrepancyInClean: CleanBuildDescrepancy | undefined, message: string) {
-                if (descrepancyInClean === undefined) {
-                    assert.equal(incrementalText, cleanText, message);
-                    return;
-                }
-                switch (descrepancyInClean) {
-                    case CleanBuildDescrepancy.CleanFileTextDifferent:
-                        assert.isDefined(incrementalText, `Incremental file should be present:: ${message}`);
-                        assert.isDefined(cleanText, `Clean file should be present present:: ${message}`);
-                        assert.notEqual(incrementalText, cleanText, message);
-                        return;
-                    case CleanBuildDescrepancy.CleanFilePresent:
-                        assert.isUndefined(incrementalText, `Incremental file should be absent:: ${message}`);
-                        assert.isDefined(cleanText, `Clean file should be present:: ${message}`);
-                        return;
-                    default:
-                        Debug.assertNever(descrepancyInClean);
-                }
-            }
-
-            function verifyMapLike<T>(incremental: MapLike<T> | undefined, clean: MapLike<T> | undefined, verifyValue: (key: string, incrementalValue: T, cleanValue: T) => void, message: string) {
-                verifyPresenceAbsence(incremental, clean, `Incremental and clean presence should match:: ${message}`);
-                if (!incremental) return;
-                const incrementalMap = new Map(getEntries(incremental));
-                const cleanMap = new Map(getEntries(clean!));
-                assert.equal(incrementalMap.size, cleanMap.size, `Incremental and clean size of map should match:: ${message}, Incremental keys: ${arrayFrom(incrementalMap.keys())} Clean: ${arrayFrom(cleanMap.keys())}${TestFSWithWatch.getDiffInKeys(incrementalMap, arrayFrom(cleanMap.keys()))}`);
-                cleanMap.forEach((cleanValue, key) => {
-                    assert.isTrue(incrementalMap.has(key), `Expected to contain ${key} in incremental map:: ${message}, Incremental keys: ${arrayFrom(incrementalMap.keys())}`);
-                    verifyValue(key, incrementalMap.get(key)!, cleanValue);
-                });
-            }
+    function verifyTscEditDiscrepancies({
+        index, scenario, subScenario, commandLineArgs,
+        discrepancyExplanation, baselines,
+        modifyFs, editFs, baseFs, newSys
+    }: VerifyTscEditDiscrepanciesInput): string[] | undefined {
+        const sys = testTscCompile({
+            scenario,
+            subScenario,
+            fs: () => baseFs.makeReadonly(),
+            commandLineArgs,
+            modifyFs: fs => {
+                if (modifyFs) modifyFs(fs);
+                editFs(fs);
+            },
+            disableUseFileVersionAsSignature: true,
         });
-    }
+        let headerAdded = false;
+        for (const outputFile of arrayFrom(sys.writtenFiles.keys())) {
+            const cleanBuildText = sys.readFile(outputFile);
+            const incrementalBuildText = newSys.readFile(outputFile);
+            if (isBuildInfoFile(outputFile)) {
+                // Check only presence and absence and not text as we will do that for readable baseline
+                if (!sys.fileExists(`${outputFile}.readable.baseline.txt`)) addBaseline(`Readable baseline not present in clean build:: File:: ${outputFile}`);
+                if (!newSys.fileExists(`${outputFile}.readable.baseline.txt`)) addBaseline(`Readable baseline not present in incremental build:: File:: ${outputFile}`);
+                verifyPresenceAbsence(incrementalBuildText, cleanBuildText, `Incremental and clean tsbuildinfo file presence differs:: File:: ${outputFile}`);
+            }
+            else if (!fileExtensionIs(outputFile, ".tsbuildinfo.readable.baseline.txt")) {
+                verifyTextEqual(incrementalBuildText, cleanBuildText, `File: ${outputFile}`);
+            }
+            else if (incrementalBuildText !== cleanBuildText) {
+                // Verify build info without affectedFilesPendingEmit
+                const { buildInfo: incrementalBuildInfo, readableBuildInfo: incrementalReadableBuildInfo } = getBuildInfoForIncrementalCorrectnessCheck(incrementalBuildText);
+                const { buildInfo: cleanBuildInfo, readableBuildInfo: cleanReadableBuildInfo } = getBuildInfoForIncrementalCorrectnessCheck(cleanBuildText);
+                verifyTextEqual(incrementalBuildInfo, cleanBuildInfo, `TsBuild info text without affectedFilesPendingEmit:: ${outputFile}::`);
+                // Verify file info sigantures
+                verifyMapLike(
+                    incrementalReadableBuildInfo?.program?.fileInfos,
+                    cleanReadableBuildInfo?.program?.fileInfos,
+                    (key, incrementalFileInfo, cleanFileInfo) => {
+                        if (incrementalFileInfo.signature !== cleanFileInfo.signature && incrementalFileInfo.signature !== incrementalFileInfo.version) {
+                            return [
+                                `Incremental signature is neither dts signature nor file version for File:: ${key}`,
+                                `Incremental:: ${JSON.stringify(incrementalFileInfo, /*replacer*/ undefined, 2)}`,
+                                `Clean:: ${JSON.stringify(cleanFileInfo, /*replacer*/ undefined, 2)}`
+                            ];
+                        }
+                    },
+                    `FileInfos:: File:: ${outputFile}`
+                );
+                // Verify exportedModulesMap
+                verifyMapLike(
+                    incrementalReadableBuildInfo?.program?.exportedModulesMap,
+                    cleanReadableBuildInfo?.program?.exportedModulesMap,
+                    (key, incrementalReferenceSet, cleanReferenceSet) => {
+                        if (!arrayIsEqualTo(incrementalReferenceSet, cleanReferenceSet) && !arrayIsEqualTo(incrementalReferenceSet, incrementalReadableBuildInfo!.program!.referencedMap![key])) {
+                            return [
+                                `Incremental Reference set is neither from dts nor files reference map for File:: ${key}::`,
+                                `Incremental:: ${JSON.stringify(incrementalReferenceSet, /*replacer*/ undefined, 2)}`,
+                                `Clean:: ${JSON.stringify(cleanReferenceSet, /*replacer*/ undefined, 2)}`,
+                                `IncrementalReferenceMap:: ${JSON.stringify(incrementalReadableBuildInfo!.program!.referencedMap![key], /*replacer*/ undefined, 2)}`,
+                                `CleanReferenceMap:: ${JSON.stringify(cleanReadableBuildInfo!.program!.referencedMap![key], /*replacer*/ undefined, 2)}`,
+                            ];
+                        }
+                    },
+                    `exportedModulesMap:: File:: ${outputFile}`
+                );
+                // Verify that incrementally pending affected file emit are in clean build since clean build can contain more files compared to incremental depending of noEmitOnError option
+                if (incrementalReadableBuildInfo?.program?.affectedFilesPendingEmit) {
+                    if (cleanReadableBuildInfo?.program?.affectedFilesPendingEmit === undefined) {
+                        addBaseline(
+                            `Incremental build contains affectedFilesPendingEmit, clean build does not have it: ${outputFile}::`,
+                            `Incremental buildInfoText:: ${incrementalBuildText}`,
+                            `Clean buildInfoText:: ${cleanBuildText}`
+                        );
+                    }
+                    let expectedIndex = 0;
+                    incrementalReadableBuildInfo.program.affectedFilesPendingEmit.forEach(([actualFile]) => {
+                        expectedIndex = findIndex(cleanReadableBuildInfo!.program!.affectedFilesPendingEmit!, ([expectedFile]) => actualFile === expectedFile, expectedIndex);
+                        if (expectedIndex === -1) {
+                            addBaseline(
+                                `Incremental build contains ${actualFile} file as pending emit, clean build does not have it: ${outputFile}::`,
+                                `Incremental buildInfoText:: ${incrementalBuildText}`,
+                                `Clean buildInfoText:: ${cleanBuildText}`
+                            );
+                        }
+                        expectedIndex++;
+                    });
+                }
+            }
+        }
+        if (!headerAdded && discrepancyExplanation) addBaseline("*** Supplied discrepancy explanation but didnt file any difference");
+        return baselines;
 
-    function verifyPresenceAbsence<T>(actual: T | undefined, expected: T | undefined, message: string) {
-        (expected !== undefined ? assert.isDefined : assert.isUndefined)(actual, message);
+        function verifyTextEqual(incrementalText: string | undefined, cleanText: string | undefined, message: string) {
+            if (incrementalText !== cleanText) writeNotEqual(incrementalText, cleanText, message);
+        }
+
+        function verifyMapLike<T>(incremental: MapLike<T> | undefined, clean: MapLike<T> | undefined, verifyValue: (key: string, incrementalValue: T, cleanValue: T) => string[] | undefined, message: string) {
+            verifyPresenceAbsence(incremental, clean, `Incremental and clean do not match:: ${message}`);
+            if (!incremental || !clean) return;
+            const incrementalMap = new Map(getEntries(incremental));
+            const cleanMap = new Map(getEntries(clean));
+            if (incrementalMap.size !== cleanMap.size) {
+                addBaseline(
+                    `Incremental and clean size of maps do not match:: ${message}`,
+                    `Incremental: ${JSON.stringify(incremental, /*replacer*/ undefined, 2)}`,
+                    `Clean: ${JSON.stringify(clean, /*replacer*/ undefined, 2)}`,
+                );
+                return;
+            }
+            cleanMap.forEach((cleanValue, key) => {
+                const incrementalValue = incrementalMap.get(key);
+                if (!incrementalValue) {
+                    addBaseline(
+                        `Incremental does not contain ${key} which is present in clean:: ${message}`,
+                        `Incremental: ${JSON.stringify(incremental, /*replacer*/ undefined, 2)}`,
+                        `Clean: ${JSON.stringify(clean, /*replacer*/ undefined, 2)}`,
+                    );
+                }
+                else {
+                    const result = verifyValue(key, incrementalMap.get(key)!, cleanValue);
+                    if (result) addBaseline(...result);
+                }
+            });
+        }
+
+        function verifyPresenceAbsence<T>(actual: T | undefined, expected: T | undefined, message: string) {
+            if (expected === undefined) {
+                if (actual === undefined) return;
+            }
+            else {
+                if (actual !== undefined) return;
+            }
+            writeNotEqual(actual, expected, message);
+        }
+
+        function writeNotEqual<T>(actual: T | undefined, expected: T | undefined, message: string) {
+            addBaseline(
+                message,
+                "CleanBuild:",
+                isString(expected) ? expected : JSON.stringify(expected),
+                "IncrementalBuild:",
+                isString(actual) ? actual : JSON.stringify(actual),
+            );
+        }
+
+        function addBaseline(...text: string[]) {
+            if (!baselines || !headerAdded) {
+                (baselines ||= []).push(`${index}:: ${subScenario}`, ...(discrepancyExplanation?.()|| ["*** Needs explanation"]));
+                headerAdded = true;
+            }
+            baselines.push(...text);
+        }
     }
 
     function getBuildInfoForIncrementalCorrectnessCheck(text: string | undefined): {
@@ -425,7 +463,7 @@ interface Symbol {
         if (!text) return { buildInfo: text };
         const readableBuildInfo = JSON.parse(text) as ReadableBuildInfo;
         let sanitizedFileInfos: MapLike<BuilderState.FileInfo> | undefined;
-        if (readableBuildInfo.program) {
+        if (readableBuildInfo.program?.fileInfos) {
             sanitizedFileInfos = {};
             for (const id in readableBuildInfo.program.fileInfos) {
                 if (hasProperty(readableBuildInfo.program.fileInfos, id)) {
@@ -461,34 +499,22 @@ interface Symbol {
         modifyFs: (fs: vfs.FileSystem) => void;
         subScenario: string;
         commandLineArgs?: readonly string[];
-        cleanBuildDiscrepancies?: () => ESMap<string, CleanBuildDescrepancy>;
+        /** An array of lines to be printed in order when a discrepancy is detected */
+        discrepancyExplanation?: () => readonly string[];
     }
 
-    export interface VerifyTscWithEditsInput extends VerifyTscWithEditsWorkerInput {
-        baselineIncremental?: boolean;
-    }
-    export interface VerifyTscWithEditsWorkerInput extends TestTscCompile {
+    export interface VerifyTscWithEditsInput extends TestTscCompile {
         edits: TestTscEdit[];
     }
 
     /**
      * Verify non watch tsc invokcation after each edit
      */
-    export function verifyTscWithEdits(input: VerifyTscWithEditsInput) {
-        verifyTscWithEditsWorker(input);
-        if (input.baselineIncremental) {
-            verifyTscWithEditsWorker({
-                ...input,
-                subScenario: `${input.subScenario} with incremental`,
-                commandLineArgs: [...input.commandLineArgs, "--incremental"],
-            });
-        }
-    }
-    function verifyTscWithEditsWorker({
+    export function verifyTscWithEdits({
         subScenario, fs, scenario, commandLineArgs,
         baselineSourceMap, modifyFs, baselineReadFileCalls, baselinePrograms,
         edits
-    }: VerifyTscWithEditsWorkerInput) {
+    }: VerifyTscWithEditsInput) {
         describe(`tsc ${commandLineArgs.join(" ")} ${scenario}:: ${subScenario} serializedEdits`, () => {
             let sys: TscCompileSystem;
             let baseFs: vfs.FileSystem;
@@ -528,35 +554,43 @@ interface Symbol {
                 sys = undefined!;
                 editsSys = undefined!;
             });
-            describe("tsc invocation after edit", () => {
-                verifyTscBaseline(() => ({
-                    baseLine: () => {
-                        const { file, text } = sys.baseLine();
-                        const texts: string[] = [text];
-                        editsSys.forEach((sys, index) => {
-                            const incrementalScenario = edits[index];
-                            texts.push("");
-                            texts.push(`Change:: ${incrementalScenario.subScenario}`);
-                            texts.push(sys.baseLine().text);
-                        });
-                        return { file, text: texts.join("\r\n") };
-                    }
-                }));
-            });
-            describe("tsc invocation after edit and clean build correctness", () => {
-                edits.forEach(({ commandLineArgs: editCommandLineArgs, subScenario, cleanBuildDiscrepancies }, index) => verifyTscEditCorrectness(() => ({
-                    scenario,
-                    baseFs,
-                    newSys: editsSys[index],
-                    commandLineArgs: editCommandLineArgs || commandLineArgs,
-                    cleanBuildDiscrepancies,
-                    editFs: fs => {
-                        for (let i = 0; i <= index; i++) {
-                            edits[i].modifyFs(fs);
-                        }
-                    },
-                    modifyFs,
-                }), index, subScenario));
+            verifyTscBaseline(() => ({
+                baseLine: () => {
+                    const { file, text } = sys.baseLine();
+                    const texts: string[] = [text];
+                    editsSys.forEach((sys, index) => {
+                        const incrementalScenario = edits[index];
+                        texts.push("");
+                        texts.push(`Change:: ${incrementalScenario.subScenario}`);
+                        texts.push(sys.baseLine().text);
+                    });
+                    return { file, text: texts.join("\r\n") };
+                }
+            }));
+            it("tsc invocation after edit and clean build correctness", () => {
+                let baselines: string[] | undefined;
+                for (let index = 0; index < edits.length; index++) {
+                    baselines = verifyTscEditDiscrepancies({
+                        index,
+                        scenario,
+                        subScenario: edits[index].subScenario,
+                        baselines,
+                        baseFs,
+                        newSys: editsSys[index],
+                        commandLineArgs: edits[index].commandLineArgs || commandLineArgs,
+                        discrepancyExplanation: edits[index].discrepancyExplanation,
+                        editFs: fs => {
+                            for (let i = 0; i <= index; i++) {
+                                edits[i].modifyFs(fs);
+                            }
+                        },
+                        modifyFs
+                    });
+                }
+                Harness.Baseline.runBaseline(
+                    `${isBuild(commandLineArgs) ? "tsbuild" : "tsc"}/${scenario}/${subScenario.split(" ").join("-")}-discrepancies.js`,
+                    baselines ? baselines.join("\r\n") : null // eslint-disable-line no-null/no-null
+                );
             });
         });
     }

src/testRunner/unittests/tsbuild/noEmitOnError.ts

@@ -8,40 +8,81 @@ namespace ts {
             projFs = undefined!;
         });
 
-        function verifyNoEmitOnError(subScenario: string, fixModifyFs: TestTscEdit["modifyFs"], modifyFs?: TestTscEdit["modifyFs"]) {
-            verifyTscWithEdits({
-                scenario: "noEmitOnError",
-                subScenario,
-                fs: () => projFs,
-                modifyFs,
-                commandLineArgs: ["--b", "/src/tsconfig.json"],
-                edits: [
-                    noChangeRun,
-                    {
-                        subScenario: "Fix error",
-                        modifyFs: fixModifyFs,
-                    },
-                    noChangeRun,
-                ],
-                baselinePrograms: true,
-                baselineIncremental: true
-            });
-        }
-
-        verifyNoEmitOnError(
-            "syntax errors",
-            fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+        verifyTscWithEdits({
+            scenario: "noEmitOnError",
+            subScenario: "syntax errors",
+            fs: () => projFs,
+            commandLineArgs: ["--b", "/src/tsconfig.json"],
+            edits: [
+                noChangeRun,
+                {
+                    subScenario: "Fix error",
+                    modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
 const a = {
     lastName: 'sdsd'
-};`, "utf-8")
-        );
+};`, "utf-8"),
+                },
+                noChangeRun,
+            ],
+            baselinePrograms: true,
+        });
 
-        verifyNoEmitOnError(
-            "semantic errors",
-            fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+        verifyTscWithEdits({
+            scenario: "noEmitOnError",
+            subScenario: "syntax errors with incremental",
+            fs: () => projFs,
+            commandLineArgs: ["--b", "/src/tsconfig.json", "--incremental"],
+            edits: [
+                noChangeRun,
+                {
+                    subScenario: "Fix error",
+                    modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+const a = {
+    lastName: 'sdsd'
+};`, "utf-8"),
+                    discrepancyExplanation: noChangeWithExportsDiscrepancyRun.discrepancyExplanation,
+                },
+                noChangeWithExportsDiscrepancyRun,
+            ],
+            baselinePrograms: true,
+        });
+
+        verifyTscWithEdits({
+            scenario: "noEmitOnError",
+            subScenario: "semantic errors",
+            fs: () => projFs,
+            modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+const a: string = 10;`, "utf-8"),
+            commandLineArgs: ["--b", "/src/tsconfig.json"],
+            edits: [
+                noChangeRun,
+                {
+                    subScenario: "Fix error",
+                    modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
 const a: string = "hello";`, "utf-8"),
-            fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
-const a: string = 10;`, "utf-8")
-        );
+                },
+                noChangeRun,
+            ],
+            baselinePrograms: true,
+        });
+
+        verifyTscWithEdits({
+            scenario: "noEmitOnError",
+            subScenario: "semantic errors with incremental",
+            fs: () => projFs,
+            modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+const a: string = 10;`, "utf-8"),
+            commandLineArgs: ["--b", "/src/tsconfig.json", "--incremental"],
+            edits: [
+                noChangeWithExportsDiscrepancyRun,
+                {
+                    subScenario: "Fix error",
+                    modifyFs: fs => fs.writeFileSync("/src/src/main.ts", `import { A } from "../shared/types/db";
+const a: string = "hello";`, "utf-8"),
+                },
+                noChangeRun,
+            ],
+            baselinePrograms: true,
+        });
     });
 }

src/testRunner/unittests/tsbuild/publicApi.ts

@@ -46,7 +46,7 @@ export function f22() { } // trailing`,
                 writtenFiles.add(path);
                 return originalWriteFile.call(sys, fileName, content, writeByteOrderMark);
             };
-            const { cb, getPrograms } = commandLineCallbacks(sys, /*originalReadCall*/ undefined, originalWriteFile);
+            const { cb, getPrograms } = commandLineCallbacks(sys, /*originalReadCall*/ undefined);
             const buildHost = createSolutionBuilderHost(
                 sys,
                     /*createProgram*/ undefined,

src/testRunner/unittests/tsbuild/sample.ts

@@ -246,7 +246,7 @@ namespace ts {
                     )
                 );
 
-                const host = createSolutionBuilderHost(system);
+                const host = createSolutionBuilderHostForBaseline(system);
                 const builder = createSolutionBuilder(host, [testsConfig.path], {});
                 baseline.push("Input::");
                 baselineState();
@@ -317,7 +317,7 @@ namespace ts {
                     )
                 );
 
-                const host = createSolutionBuilderHost(system);
+                const host = createSolutionBuilderHostForBaseline(system);
                 const builder = createSolutionBuilder(host, [testsConfig.path], { dry: false, force: false, verbose: false });
                 builder.build();
                 baselineState("Build of project");

src/testRunner/unittests/tsc/helpers.ts

@@ -10,7 +10,15 @@ namespace ts {
         subScenario: "no-change-run",
         modifyFs: noop
     };
+    export const noChangeWithExportsDiscrepancyRun: TestTscEdit = {
+        ...noChangeRun,
+        discrepancyExplanation: () => [
+            "Incremental build did not emit and has .ts as signature so exports has all imported modules/referenced files",
+            "Clean build always uses d.ts for signature for testing thus does not contain non exported modules/referenced files that arent needed"
+        ]
+    };
     export const noChangeOnlyRuns = [noChangeRun];
+    export const noChangeWithExportsDiscrepancyOnlyRuns = [noChangeWithExportsDiscrepancyRun];
 
     export interface TestTscCompile extends TestTscCompileLikeBase {
         baselineSourceMap?: boolean;
@@ -29,23 +37,22 @@ namespace ts {
         return !!(program as Program | BuilderProgram).getCompilerOptions;
     }
     export function commandLineCallbacks(
-        sys: System & { writtenFiles: ReadonlyCollection<Path>; },
+        sys: TscCompileSystem | tscWatch.WatchedSystem,
         originalReadCall?: System["readFile"],
-        originalWriteFile?: System["writeFile"],
     ): CommandLineCallbacks {
         let programs: CommandLineProgram[] | undefined;
 
         return {
             cb: program => {
                 if (isAnyProgram(program)) {
-                    baselineBuildInfo(program.getCompilerOptions(), sys, originalReadCall, originalWriteFile);
+                    baselineBuildInfo(program.getCompilerOptions(), sys, originalReadCall);
                     (programs || (programs = [])).push(isBuilderProgram(program) ?
                         [program.getProgram(), program] :
                         [program]
                     );
                 }
                 else {
-                    baselineBuildInfo(program.options, sys, originalReadCall, originalWriteFile);
+                    baselineBuildInfo(program.options, sys, originalReadCall);
                 }
             },
             getPrograms: () => {
@@ -122,16 +129,22 @@ ${patch ? vfs.formatPatch(patch) : ""}`
         const originalWriteFile = sys.writeFile;
         sys.writeFile = (fileName, content, writeByteOrderMark) => {
             const path = toPathWithSystem(sys, fileName);
-            assert.isFalse(writtenFiles.has(path));
+            // When buildinfo is same for two projects,
+            // it gives error and doesnt write buildinfo but because buildInfo is written for one project,
+            // readable baseline will be written two times for those two projects with same contents and is ok
+            Debug.assert(!writtenFiles.has(path) || endsWith(path, "baseline.txt"));
             writtenFiles.add(path);
             return originalWriteFile.call(sys, fileName, content, writeByteOrderMark);
         };
-        return originalWriteFile;
     }
 
-    export function createSolutionBuilderHostForBaseline(sys: TscCompileSystem, versionToWrite?: string) {
-        makeSystemReadyForBaseline(sys, versionToWrite);
-        const { cb } = commandLineCallbacks(sys);
+    export function createSolutionBuilderHostForBaseline(
+        sys: TscCompileSystem | tscWatch.WatchedSystem,
+        versionToWrite?: string,
+        originalRead?: (TscCompileSystem | tscWatch.WatchedSystem)["readFile"]
+    ) {
+        if (sys instanceof fakes.System) makeSystemReadyForBaseline(sys, versionToWrite);
+        const { cb } = commandLineCallbacks(sys, originalRead);
         const host = createSolutionBuilderHost(sys,
             /*createProgram*/ undefined,
             createDiagnosticReporter(sys, /*pretty*/ true),
@@ -155,7 +168,7 @@ ${patch ? vfs.formatPatch(patch) : ""}`
         });
 
         function commandLineCompile(sys: TscCompileSystem) {
-            const originalWriteFile = makeSystemReadyForBaseline(sys);
+            makeSystemReadyForBaseline(sys);
             actualReadFileMap = {};
             const originalReadFile = sys.readFile;
             sys.readFile = path => {
@@ -166,7 +179,7 @@ ${patch ? vfs.formatPatch(patch) : ""}`
                 return originalReadFile.call(sys, path);
             };
 
-            const result = commandLineCallbacks(sys, originalReadFile, originalWriteFile);
+            const result = commandLineCallbacks(sys, originalReadFile);
             executeCommandLine(
                 sys,
                 result.cb,

src/testRunner/unittests/tsc/incremental.ts

@@ -90,7 +90,7 @@ namespace ts {
                     commandLineArgs: ["--incremental", "-p", "src"],
                     modifyFs,
                     edits: [
-                        noChangeRun,
+                        noChangeWithExportsDiscrepancyRun,
                         {
                             subScenario: "incremental-declaration-doesnt-change",
                             modifyFs: fixModifyFs
@@ -123,15 +123,20 @@ const a: string = 10;`, "utf-8"),
             verifyNoEmitChanges({ composite: true });
 
             function verifyNoEmitChanges(compilerOptions: CompilerOptions) {
+                const discrepancyIfNoDtsEmit = getEmitDeclarations(compilerOptions) ?
+                    undefined :
+                    noChangeWithExportsDiscrepancyRun.discrepancyExplanation;
                 const noChangeRunWithNoEmit: TestTscEdit = {
                     ...noChangeRun,
                     subScenario: "No Change run with noEmit",
                     commandLineArgs: ["--p", "src/project", "--noEmit"],
+                    discrepancyExplanation: discrepancyIfNoDtsEmit,
                 };
                 const noChangeRunWithEmit: TestTscEdit = {
                     ...noChangeRun,
                     subScenario: "No Change run with emit",
                     commandLineArgs: ["--p", "src/project"],
+                    discrepancyExplanation: discrepancyIfNoDtsEmit,
                 };
                 let optionsString = "";
                 for (const key in compilerOptions) {
@@ -152,10 +157,12 @@ const a: string = 10;`, "utf-8"),
                             subScenario: "Introduce error but still noEmit",
                             commandLineArgs: ["--p", "src/project", "--noEmit"],
                             modifyFs: fs => replaceText(fs, "/src/project/src/class.ts", "prop", "prop1"),
+                            discrepancyExplanation: getEmitDeclarations(compilerOptions) ? noChangeWithExportsDiscrepancyRun.discrepancyExplanation : undefined,
                         },
                         {
                             subScenario: "Fix error and emit",
                             modifyFs: fs => replaceText(fs, "/src/project/src/class.ts", "prop1", "prop"),
+                            discrepancyExplanation: discrepancyIfNoDtsEmit
                         },
                         noChangeRunWithEmit,
                         noChangeRunWithNoEmit,
@@ -164,6 +171,7 @@ const a: string = 10;`, "utf-8"),
                         {
                             subScenario: "Introduce error and emit",
                             modifyFs: fs => replaceText(fs, "/src/project/src/class.ts", "prop", "prop1"),
+                            discrepancyExplanation: discrepancyIfNoDtsEmit
                         },
                         noChangeRunWithEmit,
                         noChangeRunWithNoEmit,
@@ -173,6 +181,7 @@ const a: string = 10;`, "utf-8"),
                             subScenario: "Fix error and no emit",
                             commandLineArgs: ["--p", "src/project", "--noEmit"],
                             modifyFs: fs => replaceText(fs, "/src/project/src/class.ts", "prop1", "prop"),
+                            discrepancyExplanation: noChangeWithExportsDiscrepancyRun.discrepancyExplanation,
                         },
                         noChangeRunWithEmit,
                         noChangeRunWithNoEmit,
@@ -196,6 +205,7 @@ const a: string = 10;`, "utf-8"),
                         {
                             subScenario: "Fix error and no emit",
                             modifyFs: fs => replaceText(fs, "/src/project/src/class.ts", "prop1", "prop"),
+                            discrepancyExplanation: noChangeWithExportsDiscrepancyRun.discrepancyExplanation
                         },
                         noChangeRunWithEmit,
                     ],
@@ -352,11 +362,10 @@ declare global {
                 {
                     subScenario: "Add class3 to project1 and build it",
                     modifyFs: fs => fs.writeFileSync("/src/projects/project1/class3.ts", `class class3 {}`, "utf-8"),
-                    cleanBuildDiscrepancies: () => new Map<string, CleanBuildDescrepancy>([
-                        // Ts buildinfo will not be updated in incremental build so it will have semantic diagnostics cached from previous build
-                        // But in clean build because of global diagnostics, semantic diagnostics are not queried so not cached in tsbuildinfo
-                        ["/src/projects/project2/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent]
-                    ]),
+                    discrepancyExplanation: () => [
+                        "Ts buildinfo will not be updated in incremental build so it will have semantic diagnostics cached from previous build",
+                        "But in clean build because of global diagnostics, semantic diagnostics are not queried so not cached in tsbuildinfo",
+                    ],
                 },
                 {
                     subScenario: "Add output of class3",
@@ -372,11 +381,10 @@ declare global {
                 {
                     subScenario: "Delete output for class3",
                     modifyFs: fs => fs.unlinkSync("/src/projects/project1/class3.d.ts"),
-                    cleanBuildDiscrepancies: () => new Map<string, CleanBuildDescrepancy>([
-                        // Ts buildinfo willbe updated but will retain lib file errors from previous build and not others because they are emitted because of change which results in clearing their semantic diagnostics cache
-                        // But in clean build because of global diagnostics, semantic diagnostics are not queried so not cached in tsbuildinfo
-                        ["/src/projects/project2/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent]
-                    ]),
+                    discrepancyExplanation: () => [
+                        "Ts buildinfo will be updated but will retain lib file errors from previous build and not others because they are emitted because of change which results in clearing their semantic diagnostics cache",
+                        "But in clean build because of global diagnostics, semantic diagnostics are not queried so not cached in tsbuildinfo",
+                    ],
                 },
                 {
                     subScenario: "Create output for class3",

src/testRunner/unittests/tscWatch/helpers.ts

@@ -171,9 +171,10 @@ namespace ts.tscWatch {
     export interface Baseline extends BaselineBase, CommandLineCallbacks {
     }
 
-    export function createBaseline(system: WatchedSystem, modifySystem?: (sys: WatchedSystem) => void): Baseline {
+    export function createBaseline(system: WatchedSystem, modifySystem?: (sys: WatchedSystem, originalRead: WatchedSystem["readFile"]) => void): Baseline {
+        const originalRead = system.readFile;
         const initialSys = fakes.patchHostForBuildInfoReadWrite(system);
-        modifySystem?.(initialSys);
+        modifySystem?.(initialSys, originalRead);
         const sys = TestFSWithWatch.changeToHostTrackingWrittenFiles(initialSys);
         const baseline: string[] = [];
         baseline.push("Input::");
@@ -410,30 +411,32 @@ namespace ts.tscWatch {
         sys.writeFile(file, content.replace(searchValue, replaceValue));
     }
 
-    export function createSolutionBuilder(system: WatchedSystem, rootNames: readonly string[], defaultOptions?: BuildOptions) {
-        const host = createSolutionBuilderHost(system);
-        return ts.createSolutionBuilder(host, rootNames, defaultOptions || {});
+    export function createSolutionBuilder(system: WatchedSystem, rootNames: readonly string[], originalRead?: WatchedSystem["readFile"]) {
+        const host = createSolutionBuilderHostForBaseline(system, /*versionToWrite*/ undefined, originalRead);
+        return ts.createSolutionBuilder(host, rootNames, {});
     }
 
     export function ensureErrorFreeBuild(host: WatchedSystem, rootNames: readonly string[]) {
         // ts build should succeed
-        const solutionBuilder = createSolutionBuilder(host, rootNames, {});
-        solutionBuilder.build();
+        solutionBuildWithBaseline(host, rootNames);
         assert.equal(host.getOutput().length, 0, JSON.stringify(host.getOutput(), /*replacer*/ undefined, " "));
     }
 
-    export function createSystemWithSolutionBuild(solutionRoots: readonly string[], files: readonly TestFSWithWatch.FileOrFolderOrSymLink[], params?: TestFSWithWatch.TestServerHostCreationParameters) {
-        const sys = createWatchedSystem(files, params);
+    export function solutionBuildWithBaseline(sys: WatchedSystem, solutionRoots: readonly string[], originalRead?: WatchedSystem["readFile"]) {
         const originalReadFile = sys.readFile;
         const originalWrite = sys.write;
         const originalWriteFile = sys.writeFile;
         const solutionBuilder = createSolutionBuilder(TestFSWithWatch.changeToHostTrackingWrittenFiles(
             fakes.patchHostForBuildInfoReadWrite(sys)
-        ), solutionRoots, {});
+        ), solutionRoots, originalRead);
         solutionBuilder.build();
         sys.readFile = originalReadFile;
         sys.write = originalWrite;
         sys.writeFile = originalWriteFile;
         return sys;
     }
+
+    export function createSystemWithSolutionBuild(solutionRoots: readonly string[], files: readonly TestFSWithWatch.FileOrFolderOrSymLink[], params?: TestFSWithWatch.TestServerHostCreationParameters) {
+        return solutionBuildWithBaseline(createWatchedSystem(files, params), solutionRoots);
+    }
 }

src/testRunner/unittests/tscWatch/sourceOfProjectReferenceRedirect.ts

@@ -10,10 +10,8 @@ namespace ts.tscWatch {
         function verifyWatch({ files, config, subScenario }: VerifyWatchInput, alreadyBuilt: boolean) {
             const { sys, baseline, oldSnap, cb, getPrograms } = createBaseline(
                 createWatchedSystem(files),
-                alreadyBuilt ? sys => {
-                    const solutionBuilder = createSolutionBuilder(sys, [config], {});
-                    solutionBuilder.build();
-                    solutionBuilder.close();
+                alreadyBuilt ? (sys, originalRead) => {
+                    solutionBuildWithBaseline(sys, [config], originalRead);
                     sys.clearOutput();
                 } : undefined
             );

src/testRunner/unittests/tsserver/projectReferences.ts

@@ -1278,8 +1278,7 @@ bar;`
                 const files = [solnConfig, sharedConfig, sharedIndex, sharedPackage, appConfig, appBar, appIndex, sharedSymlink, libFile];
                 const host = createServerHost(files);
                 if (built) {
-                    const solutionBuilder = tscWatch.createSolutionBuilder(host, [solnConfig.path], {});
-                    solutionBuilder.build();
+                    tscWatch.solutionBuildWithBaseline(host, [solnConfig.path]);
                     host.clearOutput();
                 }
                 const session = createSession(host, { logger: createLoggerWithInMemoryLogs() });