From 4da2e5eda30eec7af1bf537c2f4b4f8352d5eb70 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Thu, 12 Apr 2018 19:31:07 +1000 Subject: [PATCH 1/8] fix: do not match MySymbol.import("mod") --- lib/typescriptServices.d.ts | 1 + src/compiler/scanner.ts | 3 +++ src/compiler/types.ts | 1 + src/harness/unittests/services/preProcessFile.ts | 13 +++++++++++++ src/services/preProcess.ts | 5 ++++- 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/typescriptServices.d.ts b/lib/typescriptServices.d.ts index ecf433425d9..c0d2a7db297 100644 --- a/lib/typescriptServices.d.ts +++ b/lib/typescriptServices.d.ts @@ -2929,6 +2929,7 @@ declare namespace ts { getTokenText(): string; getTokenValue(): string; hasExtendedUnicodeEscape(): boolean; + hasPrecedingDot(): boolean; hasPrecedingLineBreak(): boolean; isIdentifier(): boolean; isReservedWord(): boolean; diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 43320b453c4..c1e7dbb299d 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -22,6 +22,7 @@ namespace ts { getTokenText(): string; getTokenValue(): string; hasExtendedUnicodeEscape(): boolean; + hasPrecedingDot(): boolean; hasPrecedingLineBreak(): boolean; isIdentifier(): boolean; isReservedWord(): boolean; @@ -833,6 +834,7 @@ namespace ts { getTokenText: () => text.substring(tokenPos, pos), getTokenValue: () => tokenValue, hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, + hasPrecedingDot: () => (tokenFlags & TokenFlags.PrecedingDot) !== 0, hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, @@ -1469,6 +1471,7 @@ namespace ts { pos++; return token = SyntaxKind.MinusToken; case CharacterCodes.dot: + tokenFlags |= TokenFlags.PrecedingDot; if (isDigit(text.charCodeAt(pos + 1))) { tokenValue = scanNumber(); return token = SyntaxKind.NumericLiteral; diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 987d403e7cd..a2eae1df6d5 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -1585,6 +1585,7 @@ namespace ts { BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000` OctalSpecifier = 1 << 8, // e.g. `0o777` ContainsSeparator = 1 << 9, // e.g. `0b1100_0101` + PrecedingDot = 1 << 10, BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier, NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier | ContainsSeparator } diff --git a/src/harness/unittests/services/preProcessFile.ts b/src/harness/unittests/services/preProcessFile.ts index 1e13bc3e345..f68af84f759 100644 --- a/src/harness/unittests/services/preProcessFile.ts +++ b/src/harness/unittests/services/preProcessFile.ts @@ -59,6 +59,19 @@ describe("PreProcessFile:", () => { }); }), + it("Do not return reference path of non-imports", () => { + test("Quill.import('delta');", + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [], + typeReferenceDirectives: [], + ambientExternalModules: undefined, + isLibFile: false + }); + }), + it("Correctly return imported files", () => { test("import i1 = require(\"r1.ts\"); import i2 =require(\"r2.ts\"); import i3= require(\"r3.ts\"); import i4=require(\"r4.ts\"); import i5 = require (\"r5.ts\");", /*readImportFile*/ true, diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index fb138a87866..95bcbbe9bc6 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -78,7 +78,7 @@ namespace ts { */ function tryConsumeImport(): boolean { let token = scanner.getToken(); - if (token === SyntaxKind.ImportKeyword) { + if (token === SyntaxKind.ImportKeyword && !scanner.hasPrecedingDot()) { token = nextToken(); if (token === SyntaxKind.OpenParenToken) { token = nextToken(); @@ -293,6 +293,9 @@ namespace ts { // export import i = require("mod") // (for JavaScript files) require("mod") + // Do not look for: + // AnySymbol.import("mod") + while (true) { if (scanner.getToken() === SyntaxKind.EndOfFileToken) { break; From ee429ef2ea4ee83c7979857cc1ddb11e94fb3474 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Fri, 13 Apr 2018 11:39:29 +1000 Subject: [PATCH 2/8] use state --- lib/typescriptServices.d.ts | 1 - src/compiler/scanner.ts | 3 --- src/compiler/types.ts | 1 - src/services/preProcess.ts | 4 +++- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/typescriptServices.d.ts b/lib/typescriptServices.d.ts index c0d2a7db297..ecf433425d9 100644 --- a/lib/typescriptServices.d.ts +++ b/lib/typescriptServices.d.ts @@ -2929,7 +2929,6 @@ declare namespace ts { getTokenText(): string; getTokenValue(): string; hasExtendedUnicodeEscape(): boolean; - hasPrecedingDot(): boolean; hasPrecedingLineBreak(): boolean; isIdentifier(): boolean; isReservedWord(): boolean; diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index c1e7dbb299d..43320b453c4 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -22,7 +22,6 @@ namespace ts { getTokenText(): string; getTokenValue(): string; hasExtendedUnicodeEscape(): boolean; - hasPrecedingDot(): boolean; hasPrecedingLineBreak(): boolean; isIdentifier(): boolean; isReservedWord(): boolean; @@ -834,7 +833,6 @@ namespace ts { getTokenText: () => text.substring(tokenPos, pos), getTokenValue: () => tokenValue, hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, - hasPrecedingDot: () => (tokenFlags & TokenFlags.PrecedingDot) !== 0, hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, @@ -1471,7 +1469,6 @@ namespace ts { pos++; return token = SyntaxKind.MinusToken; case CharacterCodes.dot: - tokenFlags |= TokenFlags.PrecedingDot; if (isDigit(text.charCodeAt(pos + 1))) { tokenValue = scanNumber(); return token = SyntaxKind.NumericLiteral; diff --git a/src/compiler/types.ts b/src/compiler/types.ts index a2eae1df6d5..987d403e7cd 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -1585,7 +1585,6 @@ namespace ts { BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000` OctalSpecifier = 1 << 8, // e.g. `0o777` ContainsSeparator = 1 << 9, // e.g. `0b1100_0101` - PrecedingDot = 1 << 10, BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier, NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier | ContainsSeparator } diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index 95bcbbe9bc6..3cf38eecd6e 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -13,6 +13,7 @@ namespace ts { const importedFiles: FileReference[] = []; let ambientExternalModules: { ref: FileReference, depth: number }[]; let braceNesting = 0; + let lastTokenWasDot = false; // assume that text represent an external module if it contains at least one top level import/export // ambient modules that are found inside external modules are interpreted as module augmentations let externalModule = false; @@ -25,6 +26,7 @@ namespace ts { else if (token === SyntaxKind.CloseBraceToken) { braceNesting--; } + lastTokenWasDot = token === SyntaxKind.DotToken; return token; } @@ -78,7 +80,7 @@ namespace ts { */ function tryConsumeImport(): boolean { let token = scanner.getToken(); - if (token === SyntaxKind.ImportKeyword && !scanner.hasPrecedingDot()) { + if (token === SyntaxKind.ImportKeyword && !lastTokenWasDot) { token = nextToken(); if (token === SyntaxKind.OpenParenToken) { token = nextToken(); From 02b8160b41636a3a742f5e4de0b13d9415ef2819 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Sun, 15 Apr 2018 01:23:27 +1000 Subject: [PATCH 3/8] store lastToken --- src/services/preProcess.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index 3cf38eecd6e..208d276a73c 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -13,20 +13,20 @@ namespace ts { const importedFiles: FileReference[] = []; let ambientExternalModules: { ref: FileReference, depth: number }[]; let braceNesting = 0; - let lastTokenWasDot = false; + let lastToken: SyntaxKind = null; // assume that text represent an external module if it contains at least one top level import/export // ambient modules that are found inside external modules are interpreted as module augmentations let externalModule = false; function nextToken() { const token = scanner.scan(); + lastToken = token; if (token === SyntaxKind.OpenBraceToken) { braceNesting++; } else if (token === SyntaxKind.CloseBraceToken) { braceNesting--; } - lastTokenWasDot = token === SyntaxKind.DotToken; return token; } @@ -79,6 +79,7 @@ namespace ts { * Returns true if at least one token was consumed from the stream */ function tryConsumeImport(): boolean { + const lastTokenWasDot = lastToken === SyntaxKind.DotToken; let token = scanner.getToken(); if (token === SyntaxKind.ImportKeyword && !lastTokenWasDot) { token = nextToken(); From f4fa2eb28e014ebd42f6a368ab5b91ed7a128bf5 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Sun, 15 Apr 2018 01:51:01 +1000 Subject: [PATCH 4/8] undefined instead of null --- src/services/preProcess.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index 208d276a73c..f0958193ceb 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -13,7 +13,7 @@ namespace ts { const importedFiles: FileReference[] = []; let ambientExternalModules: { ref: FileReference, depth: number }[]; let braceNesting = 0; - let lastToken: SyntaxKind = null; + let lastToken: SyntaxKind; // assume that text represent an external module if it contains at least one top level import/export // ambient modules that are found inside external modules are interpreted as module augmentations let externalModule = false; From c64c1c93954ec3f13443ce08ff57d4bf93dbff4d Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Sun, 15 Apr 2018 03:30:47 +1000 Subject: [PATCH 5/8] fix up --- src/services/preProcess.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index f0958193ceb..f63f84b52ba 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -13,14 +13,12 @@ namespace ts { const importedFiles: FileReference[] = []; let ambientExternalModules: { ref: FileReference, depth: number }[]; let braceNesting = 0; - let lastToken: SyntaxKind; // assume that text represent an external module if it contains at least one top level import/export // ambient modules that are found inside external modules are interpreted as module augmentations let externalModule = false; function nextToken() { const token = scanner.scan(); - lastToken = token; if (token === SyntaxKind.OpenBraceToken) { braceNesting++; } @@ -79,9 +77,8 @@ namespace ts { * Returns true if at least one token was consumed from the stream */ function tryConsumeImport(): boolean { - const lastTokenWasDot = lastToken === SyntaxKind.DotToken; let token = scanner.getToken(); - if (token === SyntaxKind.ImportKeyword && !lastTokenWasDot) { + if (token === SyntaxKind.ImportKeyword) { token = nextToken(); if (token === SyntaxKind.OpenParenToken) { token = nextToken(); @@ -300,9 +297,14 @@ namespace ts { // AnySymbol.import("mod") while (true) { - if (scanner.getToken() === SyntaxKind.EndOfFileToken) { + const token = scanner.getToken(); + if (token === SyntaxKind.EndOfFileToken) { break; } + if (token === SyntaxKind.DotToken) { + nextToken(); // jump over anything directly following the dot + nextToken(); + } // check if at least one of alternative have moved scanner forward if (tryConsumeDeclare() || From 4dc9db1e70852277fe58da22b1797cbdd8fe50c6 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Sun, 15 Apr 2018 11:27:51 +1000 Subject: [PATCH 6/8] detect nested non-imports --- .../unittests/services/preProcessFile.ts | 13 ++++++++++++ src/services/preProcess.ts | 21 ++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/harness/unittests/services/preProcessFile.ts b/src/harness/unittests/services/preProcessFile.ts index f68af84f759..76b1e4f05a6 100644 --- a/src/harness/unittests/services/preProcessFile.ts +++ b/src/harness/unittests/services/preProcessFile.ts @@ -72,6 +72,19 @@ describe("PreProcessFile:", () => { }); }), + it("Do not return reference path of nested non-imports", () => { + test("a.b.import('c');", + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [], + typeReferenceDirectives: [], + ambientExternalModules: undefined, + isLibFile: false + }); + }), + it("Correctly return imported files", () => { test("import i1 = require(\"r1.ts\"); import i2 =require(\"r2.ts\"); import i3= require(\"r3.ts\"); import i4=require(\"r4.ts\"); import i5 = require (\"r5.ts\");", /*readImportFile*/ true, diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index f63f84b52ba..a1cb4676038 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -12,20 +12,25 @@ namespace ts { }; const importedFiles: FileReference[] = []; let ambientExternalModules: { ref: FileReference, depth: number }[]; + let lastToken: SyntaxKind; + let currentToken: SyntaxKind; let braceNesting = 0; // assume that text represent an external module if it contains at least one top level import/export // ambient modules that are found inside external modules are interpreted as module augmentations let externalModule = false; function nextToken() { - const token = scanner.scan(); - if (token === SyntaxKind.OpenBraceToken) { + if (currentToken) { + lastToken = currentToken; + } + currentToken = scanner.scan(); + if (currentToken === SyntaxKind.OpenBraceToken) { braceNesting++; } - else if (token === SyntaxKind.CloseBraceToken) { + else if (currentToken === SyntaxKind.CloseBraceToken) { braceNesting--; } - return token; + return currentToken; } function getFileReference() { @@ -77,6 +82,9 @@ namespace ts { * Returns true if at least one token was consumed from the stream */ function tryConsumeImport(): boolean { + if (lastToken === SyntaxKind.DotToken) { + return false; + } let token = scanner.getToken(); if (token === SyntaxKind.ImportKeyword) { token = nextToken(); @@ -295,16 +303,13 @@ namespace ts { // Do not look for: // AnySymbol.import("mod") + // AnySymbol.nested.import("mod") while (true) { const token = scanner.getToken(); if (token === SyntaxKind.EndOfFileToken) { break; } - if (token === SyntaxKind.DotToken) { - nextToken(); // jump over anything directly following the dot - nextToken(); - } // check if at least one of alternative have moved scanner forward if (tryConsumeDeclare() || From a3311edb4a65a9c80d43ba7beb75bef16485a054 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Sun, 15 Apr 2018 11:30:37 +1000 Subject: [PATCH 7/8] revert change --- src/services/preProcess.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index a1cb4676038..7b802301204 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -306,8 +306,7 @@ namespace ts { // AnySymbol.nested.import("mod") while (true) { - const token = scanner.getToken(); - if (token === SyntaxKind.EndOfFileToken) { + if (scanner.getToken() === SyntaxKind.EndOfFileToken) { break; } From f0057f9a333ebeba8dd4b7ee882634be83904f44 Mon Sep 17 00:00:00 2001 From: Mohamed Hegazy Date: Mon, 16 Apr 2018 11:48:04 -0700 Subject: [PATCH 8/8] Remove condition --- src/services/preProcess.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/services/preProcess.ts b/src/services/preProcess.ts index 7b802301204..6c5f6c1b2ff 100644 --- a/src/services/preProcess.ts +++ b/src/services/preProcess.ts @@ -20,9 +20,7 @@ namespace ts { let externalModule = false; function nextToken() { - if (currentToken) { - lastToken = currentToken; - } + lastToken = currentToken; currentToken = scanner.scan(); if (currentToken === SyntaxKind.OpenBraceToken) { braceNesting++;