Simplify constraint depth limiter logic (#41972)

* Explore at least 10 levels of constraints before checking for deeply nested types

* Simplify constraint depth limiter logic

* Add regression test

* Accept new baselines

src/compiler/checker.ts

@@ -336,7 +336,6 @@ namespace ts {
         let totalInstantiationCount = 0;
         let instantiationCount = 0;
         let instantiationDepth = 0;
-        let constraintDepth = 0;
         let currentNode: Node | undefined;
 
         const typeCatalog: Type[] = []; // NB: id is index + 1
@@ -11033,7 +11032,6 @@ namespace ts {
             if (type.resolvedBaseConstraint) {
                 return type.resolvedBaseConstraint;
             }
-            let nonTerminating = false;
             const stack: Type[] = [];
             return type.resolvedBaseConstraint = getTypeWithThisArgument(getImmediateBaseConstraint(type), type);
 
@@ -11042,22 +11040,16 @@ namespace ts {
                     if (!pushTypeResolution(t, TypeSystemPropertyName.ImmediateBaseConstraint)) {
                         return circularConstraintType;
                     }
-                    if (constraintDepth >= 50) {
-                        // We have reached 50 recursive invocations of getImmediateBaseConstraint and there is a
-                        // very high likelihood we're dealing with an infinite generic type that perpetually generates
-                        // new type identities as we descend into it. We stop the recursion here and mark this type
-                        // and the outer types as having circular constraints.
-                        tracing.instant(tracing.Phase.CheckTypes, "getImmediateBaseConstraint_DepthLimit", { typeId: t.id, originalTypeId: type.id, depth: constraintDepth });
-                        error(currentNode, Diagnostics.Type_instantiation_is_excessively_deep_and_possibly_infinite);
-                        nonTerminating = true;
-                        return t.immediateBaseConstraint = noConstraintType;
-                    }
                     let result;
-                    if (!isDeeplyNestedType(t, stack, stack.length)) {
+                    // We always explore at least 10 levels of nested constraints. Thereafter, we continue to explore
+                    // up to 50 levels of nested constraints provided there are no "deeply nested" types on the stack
+                    // (i.e. no types for which five instantiations have been recorded on the stack). If we reach 50
+                    // levels of nesting, we are presumably exploring a repeating pattern with a long cycle that hasn't
+                    // yet triggered the deeply nested limiter. We have no test cases that actually get to 50 levels of
+                    // nesting, so it is effectively just a safety stop.
+                    if (stack.length < 10 || stack.length < 50 && !isDeeplyNestedType(t, stack, stack.length)) {
                         stack.push(t);
-                        constraintDepth++;
                         result = computeBaseConstraint(getSimplifiedType(t, /*writing*/ false));
-                        constraintDepth--;
                         stack.pop();
                     }
                     if (!popTypeResolution()) {
@@ -11072,9 +11064,6 @@ namespace ts {
                         }
                         result = circularConstraintType;
                     }
-                    if (nonTerminating) {
-                        result = circularConstraintType;
-                    }
                     t.immediateBaseConstraint = result || noConstraintType;
                 }
                 return t.immediateBaseConstraint;
@@ -11125,10 +11114,7 @@ namespace ts {
                 }
                 if (t.flags & TypeFlags.Conditional) {
                     const constraint = getConstraintFromConditionalType(<ConditionalType>t);
-                    constraintDepth++; // Penalize repeating conditional types (this captures the recursion within getConstraintFromConditionalType and carries it forward)
-                    const result = constraint && getBaseConstraint(constraint);
-                    constraintDepth--;
-                    return result;
+                    return constraint && getBaseConstraint(constraint);
                 }
                 if (t.flags & TypeFlags.Substitution) {
                     return getBaseConstraint((<SubstitutionType>t).substitute);

tests/baselines/reference/deeplyNestedConstraints.js

@@ -0,0 +1,35 @@
+//// [deeplyNestedConstraints.ts]
+// Repro from #41931
+
+type Enum = Record<string, string | number>;
+
+type TypeMap<E extends Enum> = { [key in E[keyof E]]: number | boolean | string | number[] };
+
+class BufferPool<E extends Enum, M extends TypeMap<E>> {
+    setArray2<K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>) {
+        array.length; // Requires exploration of >5 levels of constraints
+    }
+}
+
+
+//// [deeplyNestedConstraints.js]
+"use strict";
+// Repro from #41931
+var BufferPool = /** @class */ (function () {
+    function BufferPool() {
+    }
+    BufferPool.prototype.setArray2 = function (_, array) {
+        array.length; // Requires exploration of >5 levels of constraints
+    };
+    return BufferPool;
+}());
+
+
+//// [deeplyNestedConstraints.d.ts]
+declare type Enum = Record<string, string | number>;
+declare type TypeMap<E extends Enum> = {
+    [key in E[keyof E]]: number | boolean | string | number[];
+};
+declare class BufferPool<E extends Enum, M extends TypeMap<E>> {
+    setArray2<K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>): void;
+}

tests/baselines/reference/deeplyNestedConstraints.symbols

@@ -0,0 +1,43 @@
+=== tests/cases/compiler/deeplyNestedConstraints.ts ===
+// Repro from #41931
+
+type Enum = Record<string, string | number>;
+>Enum : Symbol(Enum, Decl(deeplyNestedConstraints.ts, 0, 0))
+>Record : Symbol(Record, Decl(lib.es5.d.ts, --, --))
+
+type TypeMap<E extends Enum> = { [key in E[keyof E]]: number | boolean | string | number[] };
+>TypeMap : Symbol(TypeMap, Decl(deeplyNestedConstraints.ts, 2, 44))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 4, 13))
+>Enum : Symbol(Enum, Decl(deeplyNestedConstraints.ts, 0, 0))
+>key : Symbol(key, Decl(deeplyNestedConstraints.ts, 4, 34))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 4, 13))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 4, 13))
+
+class BufferPool<E extends Enum, M extends TypeMap<E>> {
+>BufferPool : Symbol(BufferPool, Decl(deeplyNestedConstraints.ts, 4, 93))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 6, 17))
+>Enum : Symbol(Enum, Decl(deeplyNestedConstraints.ts, 0, 0))
+>M : Symbol(M, Decl(deeplyNestedConstraints.ts, 6, 32))
+>TypeMap : Symbol(TypeMap, Decl(deeplyNestedConstraints.ts, 2, 44))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 6, 17))
+
+    setArray2<K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>) {
+>setArray2 : Symbol(BufferPool.setArray2, Decl(deeplyNestedConstraints.ts, 6, 56))
+>K : Symbol(K, Decl(deeplyNestedConstraints.ts, 7, 14))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 6, 17))
+>E : Symbol(E, Decl(deeplyNestedConstraints.ts, 6, 17))
+>_ : Symbol(_, Decl(deeplyNestedConstraints.ts, 7, 36))
+>K : Symbol(K, Decl(deeplyNestedConstraints.ts, 7, 14))
+>array : Symbol(array, Decl(deeplyNestedConstraints.ts, 7, 41))
+>Extract : Symbol(Extract, Decl(lib.es5.d.ts, --, --))
+>M : Symbol(M, Decl(deeplyNestedConstraints.ts, 6, 32))
+>K : Symbol(K, Decl(deeplyNestedConstraints.ts, 7, 14))
+>ArrayLike : Symbol(ArrayLike, Decl(lib.es5.d.ts, --, --))
+
+        array.length; // Requires exploration of >5 levels of constraints
+>array.length : Symbol(length, Decl(lib.es5.d.ts, --, --), Decl(lib.es5.d.ts, --, --))
+>array : Symbol(array, Decl(deeplyNestedConstraints.ts, 7, 41))
+>length : Symbol(length, Decl(lib.es5.d.ts, --, --), Decl(lib.es5.d.ts, --, --))
+    }
+}
+

tests/baselines/reference/deeplyNestedConstraints.types

@@ -0,0 +1,24 @@
+=== tests/cases/compiler/deeplyNestedConstraints.ts ===
+// Repro from #41931
+
+type Enum = Record<string, string | number>;
+>Enum : Record<string, string | number>
+
+type TypeMap<E extends Enum> = { [key in E[keyof E]]: number | boolean | string | number[] };
+>TypeMap : TypeMap<E>
+
+class BufferPool<E extends Enum, M extends TypeMap<E>> {
+>BufferPool : BufferPool<E, M>
+
+    setArray2<K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>) {
+>setArray2 : <K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>) => void
+>_ : K
+>array : Extract<M[K], ArrayLike<any>>
+
+        array.length; // Requires exploration of >5 levels of constraints
+>array.length : number
+>array : Extract<M[K], ArrayLike<any>>
+>length : number
+    }
+}
+

tests/cases/compiler/deeplyNestedConstraints.ts

@@ -0,0 +1,14 @@
+// @strict: true
+// @declaration: true
+
+// Repro from #41931
+
+type Enum = Record<string, string | number>;
+
+type TypeMap<E extends Enum> = { [key in E[keyof E]]: number | boolean | string | number[] };
+
+class BufferPool<E extends Enum, M extends TypeMap<E>> {
+    setArray2<K extends E[keyof E]>(_: K, array: Extract<M[K], ArrayLike<any>>) {
+        array.length; // Requires exploration of >5 levels of constraints
+    }
+}