microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-22 22:55:36 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Numeric separators (#20324)

Browse Source 
* Add support into octal and binary literals

* Add hex support

* And finally support all numeric literals and fix spelling

* Update error message

* Refactor error in scanner to take a position

* Scan no separators in escape sequences, add escape sequence tests

* More decimal tests from the spec presentation examples

* Permissive scanning of excess separators

* Remove unnecessary assignment

* Make code easier to follow
This commit is contained in:
Wesley Wigham
2017-12-08 20:20:18 -05:00
committed by GitHub
parent 0c2d8d28de
commit 9e51882d9c
45 changed files with 2835 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/compiler/diagnosticMessages.json
View File

@@ -3420,6 +3420,10 @@
"category": "Message",
"code": 6187
},
"Numeric separators are not allowed here.": {
"category": "Error",
"code": 6188
},
"Variable '{0}' implicitly has an '{1}' type.": {
"category": "Error",
"code": 7005

129
src/compiler/scanner.ts
View File

@@ -561,9 +561,9 @@ namespace ts {
return false;
}
function scanConflictMarkerTrivia(text: string, pos: number, error?: ErrorCallback) {
function scanConflictMarkerTrivia(text: string, pos: number, error?: (diag: DiagnosticMessage, pos?: number, len?: number) => void) {
if (error) {
error(Diagnostics.Merge_conflict_marker_encountered, mergeConflictMarkerLength);
error(Diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength);
}
const ch = text.charCodeAt(pos);
@@ -852,34 +852,86 @@ namespace ts {
scanRange,
};
function error(message: DiagnosticMessage, length?: number): void {
function error(message: DiagnosticMessage): void;
function error(message: DiagnosticMessage, errPos: number, length: number): void;
function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void {
if (onError) {
const oldPos = pos;
pos = errPos;
onError(message, length || 0);
pos = oldPos;
}
}
function scanNumberFragment(): string {
let start = pos;
let allowSeparator = false;
let result = "";
while (true) {
const ch = text.charCodeAt(pos);
if (ch === CharacterCodes._) {
tokenFlags |= TokenFlags.ContainsSeparator;
if (allowSeparator) {
allowSeparator = false;
result += text.substring(start, pos);
}
else {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
}
pos++;
start = pos;
continue;
}
if (isDigit(ch)) {
allowSeparator = true;
pos++;
continue;
}
break;
}
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
}
return result + text.substring(start, pos);
}
function scanNumber(): string {
const start = pos;
while (isDigit(text.charCodeAt(pos))) pos++;
const mainFragment = scanNumberFragment();
let decimalFragment: string;
let scientificFragment: string;
if (text.charCodeAt(pos) === CharacterCodes.dot) {
pos++;
while (isDigit(text.charCodeAt(pos))) pos++;
decimalFragment = scanNumberFragment();
}
let end = pos;
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
pos++;
tokenFlags |= TokenFlags.Scientific;
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
if (isDigit(text.charCodeAt(pos))) {
pos++;
while (isDigit(text.charCodeAt(pos))) pos++;
end = pos;
}
else {
const preNumericPart = pos;
const finalFragment = scanNumberFragment();
if (!finalFragment) {
error(Diagnostics.Digit_expected);
}
else {
scientificFragment = text.substring(end, preNumericPart) + finalFragment;
end = pos;
}
}
if (tokenFlags & TokenFlags.ContainsSeparator) {
let result = mainFragment;
if (decimalFragment) {
result += "." + decimalFragment;
}
if (scientificFragment) {
result += scientificFragment;
}
return "" + +result;
}
else {
return "" + +(text.substring(start, end)); // No need to use all the fragments; no _ removal needed
}
return "" + +(text.substring(start, end));
}
function scanOctalDigits(): number {
@@ -894,23 +946,36 @@ namespace ts {
* Scans the given number of hexadecimal digits in the text,
* returning -1 if the given number is unavailable.
*/
function scanExactNumberOfHexDigits(count: number): number {
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false);
function scanExactNumberOfHexDigits(count: number, canHaveSeparators: boolean): number {
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false, canHaveSeparators);
}
/**
* Scans as many hexadecimal digits as are available in the text,
* returning -1 if the given number of digits was unavailable.
*/
function scanMinimumNumberOfHexDigits(count: number): number {
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true);
function scanMinimumNumberOfHexDigits(count: number, canHaveSeparators: boolean): number {
return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true, canHaveSeparators);
}
function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean): number {
function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean, canHaveSeparators: boolean): number {
let digits = 0;
let value = 0;
let allowSeparator = false;
while (digits < minCount || scanAsManyAsPossible) {
const ch = text.charCodeAt(pos);
if (canHaveSeparators && ch === CharacterCodes._) {
tokenFlags |= TokenFlags.ContainsSeparator;
if (allowSeparator) {
allowSeparator = false;
}
else {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
}
pos++;
continue;
}
allowSeparator = canHaveSeparators;
if (ch >= CharacterCodes._0 && ch <= CharacterCodes._9) {
value = value * 16 + ch - CharacterCodes._0;
}
@@ -929,6 +994,9 @@ namespace ts {
if (digits < minCount) {
value = -1;
}
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
}
return value;
}
@@ -1097,7 +1165,7 @@ namespace ts {
}
function scanHexadecimalEscape(numDigits: number): string {
const escapedValue = scanExactNumberOfHexDigits(numDigits);
const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false);
if (escapedValue >= 0) {
return String.fromCharCode(escapedValue);
@@ -1109,7 +1177,7 @@ namespace ts {
}
function scanExtendedUnicodeEscape(): string {
const escapedValue = scanMinimumNumberOfHexDigits(1);
const escapedValue = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
let isInvalidExtendedEscape = false;
// Validate the value of the digit
@@ -1162,7 +1230,7 @@ namespace ts {
if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) {
const start = pos;
pos += 2;
const value = scanExactNumberOfHexDigits(4);
const value = scanExactNumberOfHexDigits(4, /*canHaveSeparators*/ false);
pos = start;
return value;
}
@@ -1218,8 +1286,22 @@ namespace ts {
// For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b.
// Similarly valid octalIntegerLiteral must have at least one octal digit following o or O.
let numberOfDigits = 0;
let separatorAllowed = false;
while (true) {
const ch = text.charCodeAt(pos);
// Numeric seperators are allowed anywhere within a numeric literal, except not at the beginning, or following another separator
if (ch === CharacterCodes._) {
tokenFlags |= TokenFlags.ContainsSeparator;
if (separatorAllowed) {
separatorAllowed = false;
}
else {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
}
pos++;
continue;
}
separatorAllowed = true;
const valueOfCh = ch - CharacterCodes._0;
if (!isDigit(ch) || valueOfCh >= base) {
break;
@@ -1232,6 +1314,11 @@ namespace ts {
if (numberOfDigits === 0) {
return -1;
}
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
// Literal ends with underscore - not allowed
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
return value;
}
return value;
}
@@ -1435,7 +1522,7 @@ namespace ts {
case CharacterCodes._0:
if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
pos += 2;
let value = scanMinimumNumberOfHexDigits(1);
let value = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ true);
if (value < 0) {
error(Diagnostics.Hexadecimal_digit_expected);
value = 0;

3
src/compiler/types.ts
View File

@@ -1490,8 +1490,9 @@ namespace ts {
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
ContainsSeparator = 1 << 9, // e.g. `0b1100_0101`
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier | ContainsSeparator
}
export interface NumericLiteral extends LiteralExpression {

2
src/compiler/utilities.ts
View File

@@ -346,7 +346,7 @@ namespace ts {
export function getLiteralText(node: LiteralLikeNode, sourceFile: SourceFile) {
// If we don't need to downlevel and we can reach the original source text using
// the node's parent reference, then simply get the text as it was originally written.
if (!nodeIsSynthesized(node) && node.parent) {
if (!nodeIsSynthesized(node) && node.parent && !(isNumericLiteral(node) && node.numericLiteralFlags & TokenFlags.ContainsSeparator)) {
return getSourceTextOfNodeFromSourceFile(sourceFile, node);
}