microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-02-04 21:53:42 -06:00
Code Issues Packages Projects Releases 212 Wiki Activity

Merge pull request #901 from Microsoft/syntacticClassificationOnNewTree

Browse Source 
Switched syntactic classifier to use new tree
This commit is contained in:
Daniel Rosenwasser 2014-10-22 18:49:20 -07:00
commit fbbc4a3b59
6 changed files with 158 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/harness/fourslash.ts
View File

@ -1612,7 +1612,9 @@ module FourSlash {
private verifyClassifications(expected: { classificationType: string; text: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[]) {
if (actual.length !== expected.length) {
this.raiseError('verifyClassifications failed - expected total classifications to be ' + expected.length + ', but was ' + actual.length);
this.raiseError('verifyClassifications failed - expected total classifications to be ' + expected.length +
', but was ' + actual.length +
jsonMismatchString());
}
for (var i = 0; i < expected.length; i++) {
@ -1623,7 +1625,8 @@ module FourSlash {
if (expectedType !== actualClassification.classificationType) {
this.raiseError('verifyClassifications failed - expected classifications type to be ' +
expectedType + ', but was ' +
actualClassification.classificationType);
actualClassification.classificationType +
jsonMismatchString());
}
var expectedSpan = expectedClassification.textSpan;
@ -1635,17 +1638,25 @@ module FourSlash {
if (expectedSpan.start !== actualSpan.start() || expectedLength !== actualSpan.length()) {
this.raiseError("verifyClassifications failed - expected span of text to be " +
"{start=" + expectedSpan.start + ", length=" + expectedLength + "}, but was " +
"{start=" + actualSpan.start() + ", length=" + actualSpan.length() + "}");
"{start=" + actualSpan.start() + ", length=" + actualSpan.length() + "}" +
jsonMismatchString());
}
}
var actualText = this.activeFile.content.substr(actualSpan.start(), actualSpan.length());
if (expectedClassification.text !== actualText) {
this.raiseError('verifyClassifications failed - expected classificatied text to be ' +
this.raiseError('verifyClassifications failed - expected classified text to be ' +
expectedClassification.text + ', but was ' +
actualText);
actualText +
jsonMismatchString());
}
}
function jsonMismatchString() {
return sys.newLine +
"expected: '" + sys.newLine + JSON.stringify(expected, (k,v) => v, 2) + "'" + sys.newLine +
"actual: '" + sys.newLine + JSON.stringify(actual, (k, v) => v, 2) + "'";
}
}
public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
@ -1996,7 +2007,8 @@ module FourSlash {
var newlinePos = text.indexOf('\n');
if (newlinePos === -1) {
return text;
} else {
}
else {
if (text.charAt(newlinePos - 1) === '\r') {
newlinePos--;
}

131
src/services/services.ts
View File

@ -4826,115 +4826,99 @@ module ts {
var sourceFile = getCurrentSourceFile(fileName);
var result: ClassifiedSpan[] = [];
processElement(sourceFile.getSourceUnit());
processElement(sourceFile);
return result;
function classifyTrivia(trivia: TypeScript.ISyntaxTrivia) {
if (trivia.isComment() && span.intersectsWith(trivia.fullStart(), trivia.fullWidth())) {
function classifyComment(comment: CommentRange) {
var width = comment.end - comment.pos;
if (span.intersectsWith(comment.pos, width)) {
result.push({
textSpan: new TypeScript.TextSpan(trivia.fullStart(), trivia.fullWidth()),
textSpan: new TypeScript.TextSpan(comment.pos, width),
classificationType: ClassificationTypeNames.comment
});
}
}
function classifyTriviaList(trivia: TypeScript.ISyntaxTriviaList) {
for (var i = 0, n = trivia.count(); i < n; i++) {
classifyTrivia(trivia.syntaxTriviaAt(i));
}
}
function classifyToken(token: Node): void {
forEach(getLeadingCommentRanges(sourceFile.text, token.getFullStart()), classifyComment);
function classifyToken(token: TypeScript.ISyntaxToken) {
if (token.hasLeadingComment()) {
classifyTriviaList(token.leadingTrivia());
}
if (TypeScript.width(token) > 0) {
if (token.getWidth() > 0) {
var type = classifyTokenType(token);
if (type) {
result.push({
textSpan: new TypeScript.TextSpan(TypeScript.start(token), TypeScript.width(token)),
textSpan: new TypeScript.TextSpan(token.getStart(), token.getWidth()),
classificationType: type
});
}
}
if (token.hasTrailingComment()) {
classifyTriviaList(token.trailingTrivia());
}
forEach(getTrailingCommentRanges(sourceFile.text, token.getEnd()), classifyComment);
}
function classifyTokenType(token: TypeScript.ISyntaxToken): string {
var tokenKind = token.kind();
if (TypeScript.SyntaxFacts.isAnyKeyword(token.kind())) {
function classifyTokenType(token: Node): string {
var tokenKind = token.kind;
if (isKeyword(tokenKind)) {
return ClassificationTypeNames.keyword;
}
// Special case < and > If they appear in a generic context they are punctation,
// Special case < and > If they appear in a generic context they are punctuation,
// not operators.
if (tokenKind === TypeScript.SyntaxKind.LessThanToken || tokenKind === TypeScript.SyntaxKind.GreaterThanToken) {
var tokenParentKind = token.parent.kind();
if (tokenParentKind === TypeScript.SyntaxKind.TypeArgumentList ||
tokenParentKind === TypeScript.SyntaxKind.TypeParameterList) {
if (tokenKind === SyntaxKind.LessThanToken || tokenKind === SyntaxKind.GreaterThanToken) {
// If the node owning the token has a type argument list or type parameter list, then
// we can effectively assume that a '<' and '>' belong to those lists.
if (getTypeArgumentOrTypeParameterList(token.parent)) {
return ClassificationTypeNames.punctuation;
}
}
if (TypeScript.SyntaxFacts.isBinaryExpressionOperatorToken(tokenKind) ||
TypeScript.SyntaxFacts.isPrefixUnaryExpressionOperatorToken(tokenKind)) {
return ClassificationTypeNames.operator;
if (isPunctuation(token)) {
// the '=' in a variable declaration is special cased here.
if (token.parent.kind === SyntaxKind.BinaryExpression ||
token.parent.kind === SyntaxKind.VariableDeclaration ||
token.parent.kind === SyntaxKind.PrefixOperator ||
token.parent.kind === SyntaxKind.PostfixOperator ||
token.parent.kind === SyntaxKind.ConditionalExpression) {
return ClassificationTypeNames.operator;
}
else {
return ClassificationTypeNames.punctuation;
}
}
else if (TypeScript.SyntaxFacts.isAnyPunctuation(tokenKind)) {
return ClassificationTypeNames.punctuation;
}
else if (tokenKind === TypeScript.SyntaxKind.NumericLiteral) {
else if (tokenKind === SyntaxKind.NumericLiteral) {
return ClassificationTypeNames.numericLiteral;
}
else if (tokenKind === TypeScript.SyntaxKind.StringLiteral) {
else if (tokenKind === SyntaxKind.StringLiteral) {
return ClassificationTypeNames.stringLiteral;
}
else if (tokenKind === TypeScript.SyntaxKind.RegularExpressionLiteral) {
// TODO: we shoudl get another classification type for these literals.
else if (tokenKind === SyntaxKind.RegularExpressionLiteral) {
// TODO: we should get another classification type for these literals.
return ClassificationTypeNames.stringLiteral;
}
else if (tokenKind === TypeScript.SyntaxKind.IdentifierName) {
var current: TypeScript.ISyntaxNodeOrToken = token;
var parent = token.parent;
while (parent.kind() === TypeScript.SyntaxKind.QualifiedName) {
current = parent;
parent = parent.parent;
}
switch (parent.kind()) {
case TypeScript.SyntaxKind.SimplePropertyAssignment:
if ((<TypeScript.SimplePropertyAssignmentSyntax>parent).propertyName === token) {
return ClassificationTypeNames.identifier;
}
return;
case TypeScript.SyntaxKind.ClassDeclaration:
if ((<TypeScript.ClassDeclarationSyntax>parent).identifier === token) {
else if (tokenKind === SyntaxKind.Identifier) {
switch (token.parent.kind) {
case SyntaxKind.ClassDeclaration:
if ((<ClassDeclaration>token.parent).name === token) {
return ClassificationTypeNames.className;
}
return;
case TypeScript.SyntaxKind.TypeParameter:
if ((<TypeScript.TypeParameterSyntax>parent).identifier === token) {
case SyntaxKind.TypeParameter:
if ((<TypeParameterDeclaration>token.parent).name === token) {
return ClassificationTypeNames.typeParameterName;
}
return;
case TypeScript.SyntaxKind.InterfaceDeclaration:
if ((<TypeScript.InterfaceDeclarationSyntax>parent).identifier === token) {
case SyntaxKind.InterfaceDeclaration:
if ((<InterfaceDeclaration>token.parent).name === token) {
return ClassificationTypeNames.interfaceName;
}
return;
case TypeScript.SyntaxKind.EnumDeclaration:
if ((<TypeScript.EnumDeclarationSyntax>parent).identifier === token) {
case SyntaxKind.EnumDeclaration:
if ((<EnumDeclaration>token.parent).name === token) {
return ClassificationTypeNames.enumName;
}
return;
case TypeScript.SyntaxKind.ModuleDeclaration:
if ((<TypeScript.ModuleDeclarationSyntax>parent).name === current) {
case SyntaxKind.ModuleDeclaration:
if ((<ModuleDeclaration>token.parent).name === token) {
return ClassificationTypeNames.moduleName;
}
return;
@ -4944,19 +4928,18 @@ module ts {
}
}
function processElement(element: TypeScript.ISyntaxElement) {
function processElement(element: Node) {
// Ignore nodes that don't intersect the original span to classify.
if (!TypeScript.isShared(element) && span.intersectsWith(TypeScript.fullStart(element), TypeScript.fullWidth(element))) {
for (var i = 0, n = TypeScript.childCount(element); i < n; i++) {
var child = TypeScript.childAt(element, i);
if (child) {
if (TypeScript.isToken(child)) {
classifyToken(<TypeScript.ISyntaxToken>child);
}
else {
// Recurse into our child nodes.
processElement(child);
}
if (span.intersectsWith(element.getFullStart(), element.getFullWidth())) {
var children = element.getChildren();
for (var i = 0, n = children.length; i < n; i++) {
var child = children[i];
if (isToken(child)) {
classifyToken(child);
}
else {
// Recurse into our child nodes.
processElement(child);
}
}
}

26
src/services/utilities.ts
View File

@ -229,19 +229,35 @@ module ts {
return n.kind !== SyntaxKind.SyntaxList || n.getChildCount() !== 0;
}
export function getTypeArgumentOrTypeParameterList(node: Node): NodeArray<Node> {
if (node.kind === SyntaxKind.TypeReference || node.kind === SyntaxKind.CallExpression) {
return (<CallExpression>node).typeArguments;
}
if (isAnyFunction(node) || node.kind === SyntaxKind.ClassDeclaration || node.kind === SyntaxKind.InterfaceDeclaration) {
return (<FunctionDeclaration>node).typeParameters;
}
return undefined;
}
export function isToken(n: Node): boolean {
return n.kind >= SyntaxKind.FirstToken && n.kind <= SyntaxKind.LastToken;
}
function isKeyword(n: Node): boolean {
return n.kind >= SyntaxKind.FirstKeyword && n.kind <= SyntaxKind.LastKeyword;
}
function isWord(n: Node): boolean {
return n.kind === SyntaxKind.Identifier || isKeyword(n);
return n.kind === SyntaxKind.Identifier || isKeyword(n.kind);
}
function isPropertyName(n: Node): boolean {
return n.kind === SyntaxKind.StringLiteral || n.kind === SyntaxKind.NumericLiteral || isWord(n);
}
export function isComment(n: Node): boolean {
return n.kind === SyntaxKind.SingleLineCommentTrivia || n.kind === SyntaxKind.MultiLineCommentTrivia;
}
export function isPunctuation(n: Node): boolean {
return SyntaxKind.FirstPunctuation <= n.kind && n.kind <= SyntaxKind.LastPunctuation;
}
}

24
tests/cases/fourslash/fourslash.ts
View File

@ -575,51 +575,51 @@ module FourSlashInterface {
export module classification {
export function comment(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("comment", text, position);
return getClassification("comment", text, position);
}
export function identifier(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("identifier", text, position);
return getClassification("identifier", text, position);
}
export function keyword(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("keyword", text, position);
return getClassification("keyword", text, position);
}
export function numericLiteral(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("numericLiteral", text, position);
return getClassification("numericLiteral", text, position);
}
export function operator(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("operator", text, position);
return getClassification("operator", text, position);
}
export function stringLiteral(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("stringLiteral", text, position);
return getClassification("stringLiteral", text, position);
}
export function whiteSpace(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("whiteSpace", text, position);
return getClassification("whiteSpace", text, position);
}
export function text(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("text", text, position);
return getClassification("text", text, position);
}
export function punctuation(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("punctuation", text, position);
return getClassification("punctuation", text, position);
}
export function className(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("className", text, position);
return getClassification("className", text, position);
}
export function enumName(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("enumName", text, position);
return getClassification("enumName", text, position);
}
export function interfaceName(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
return getClassification("interfaceName", text, position);
return getClassification("interfaceName", text, position);
}
export function moduleName(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {

25
tests/cases/fourslash/syntacticClassificationsFunctionWithComments.ts Normal file
View File

@ -0,0 +1,25 @@
/// <reference path="fourslash.ts"/>
/////**
//// * This is my function.
//// * There are many like it, but this one is mine.
//// */
////function myFunction(/* x */ x: any) {
//// var y = x ? x++ : ++x;
////}
////// end of file
var firstCommentText =
"\
/**\n\
* This is my function.\n\
* There are many like it, but this one is mine.\n\
*/";
var c = classification;
verify.syntacticClassificationsAre(
c.comment(firstCommentText),
c.keyword("function"), c.text("myFunction"), c.punctuation("("), c.comment("/* x */"), c.text("x"), c.punctuation(":"), c.keyword("any"), c.punctuation(")"), c.punctuation("{"),
c.keyword("var"), c.text("y"), c.operator("="), c.text("x"), c.operator("?"), c.text("x"), c.operator("++"), c.operator(":"), c.operator("++"), c.text("x"), c.punctuation(";"),
c.punctuation("}"),
c.comment("// end of file"));

25
tests/cases/fourslash/syntacticClassificationsObjectLiteral.ts Normal file
View File

@ -0,0 +1,25 @@
/// <reference path="fourslash.ts"/>
////var v = 10e0;
////var x = {
//// p1: 1,
//// p2: 2,
//// any: 3,
//// function: 4,
//// var: 5,
//// void: void 0,
//// v: v += v,
////};
var c = classification;
verify.syntacticClassificationsAre(
c.keyword("var"), c.text("v"), c.operator("="), c.numericLiteral("10e0"), c.punctuation(";"),
c.keyword("var"), c.text("x"), c.operator("="), c.punctuation("{"),
c.text("p1"), c.punctuation(":"), c.numericLiteral("1"), c.punctuation(","),
c.text("p2"), c.punctuation(":"), c.numericLiteral("2"), c.punctuation(","),
c.text("any"), c.punctuation(":"), c.numericLiteral("3"), c.punctuation(","),
c.text("function"), c.punctuation(":"), c.numericLiteral("4"), c.punctuation(","),
c.text("var"), c.punctuation(":"), c.numericLiteral("5"), c.punctuation(","),
c.text("void"), c.punctuation(":"), c.keyword("void"), c.numericLiteral("0"), c.punctuation(","),
c.text("v"), c.punctuation(":"), c.text("v"), c.operator("+="), c.text("v"), c.punctuation(","),
c.punctuation("}"), c.punctuation(";"));