User code runner draft (#19539)

* Realworld runner draft

* Baseline tsc output instead of just checking exit code

* use latest instead of major minor pin

* Add 7 more test cases + update gitignore

* Update baselines for realworld/user tests

* Rename to user

* Do not commit lockfiles

* Add code to run user tests on CRON

* Add rest of most-dependend packages to user tests

Turns out levelup doesn't have types! So I removed that one.
This commit is contained in:
Wesley Wigham 2017-11-01 16:22:37 -07:00 committed by GitHub
parent 6a382f1436
commit ba98cbbf92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
158 changed files with 1080 additions and 2 deletions

8
.gitignore vendored
View File

@ -59,3 +59,11 @@ internal/
.idea
yarn.lock
.parallelperf.*
tests/cases/user/*/package-lock.json
tests/cases/user/*/node_modules/
tests/cases/user/*/**/*.js
tests/cases/user/*/**/*.js.map
tests/cases/user/*/**/*.d.ts
!tests/cases/user/zone.js/
!tests/cases/user/bignumber.js/
!tests/cases/user/discord.js/

View File

@ -105,6 +105,7 @@ var harnessCoreSources = [
"projectsRunner.ts",
"loggedIO.ts",
"rwcRunner.ts",
"userRunner.ts",
"test262Runner.ts",
"./parallel/shared.ts",
"./parallel/host.ts",

View File

@ -18,6 +18,7 @@
/// <reference path="fourslashRunner.ts" />
/// <reference path="projectsRunner.ts" />
/// <reference path="rwcRunner.ts" />
/// <reference path="userRunner.ts" />
/// <reference path="harness.ts" />
/// <reference path="./parallel/shared.ts" />
@ -59,6 +60,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
return new RWCRunner();
case "test262":
return new Test262BaselineRunner();
case "user":
return new UserCodeRunner();
}
ts.Debug.fail(`Unknown runner kind ${kind}`);
}
@ -175,6 +178,9 @@ function handleTestConfig() {
case "test262":
runners.push(new Test262BaselineRunner());
break;
case "user":
runners.push(new UserCodeRunner());
break;
}
}
}
@ -196,6 +202,11 @@ function handleTestConfig() {
runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
runners.push(new FourSlashRunner(FourSlashTestType.Server));
// runners.push(new GeneratedFourslashRunner());
// CRON-only tests
if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.TRAVIS_EVENT_TYPE === "cron") {
runners.push(new UserCodeRunner());
}
}
if (runUnitTests === undefined) {
runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
@ -215,8 +226,9 @@ function beginTests() {
}
}
let isWorker: boolean;
function startTestEnvironment() {
const isWorker = handleTestConfig();
isWorker = handleTestConfig();
if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
if (isWorker) {
return Harness.Parallel.Worker.start();

View File

@ -1,7 +1,7 @@
/// <reference path="harness.ts" />
type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262";
type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
type CompilerTestKind = "conformance" | "compiler";
type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";

View File

@ -92,6 +92,7 @@
"projectsRunner.ts",
"loggedIO.ts",
"rwcRunner.ts",
"userRunner.ts",
"test262Runner.ts",
"./parallel/shared.ts",
"./parallel/host.ts",

51
src/harness/userRunner.ts Normal file
View File

@ -0,0 +1,51 @@
/// <reference path="harness.ts"/>
/// <reference path="runnerbase.ts" />
class UserCodeRunner extends RunnerBase {
private static readonly testDir = "tests/cases/user/";
public enumerateTestFiles() {
return Harness.IO.getDirectories(UserCodeRunner.testDir);
}
public kind(): TestRunnerKind {
return "user";
}
/** Setup the runner's tests so that they are ready to be executed by the harness
* The first test should be a describe/it block that sets up the harness's compiler instance appropriately
*/
public initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
describe(`${this.kind()} code samples`, () => {
for (let i = 0; i < testList.length; i++) {
this.runTest(testList[i]);
}
});
}
private runTest(directoryName: string) {
describe(directoryName, () => {
const cp = require("child_process");
const path = require("path");
it("should build successfully", () => {
const cwd = path.join(__dirname, "../../", UserCodeRunner.testDir, directoryName);
const timeout = 600000; // 10 minutes
const stdio = isWorker ? "pipe" : "inherit";
const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true });
return `Exit Code: ${result.status}
Standard output:
${result.stdout.toString().replace(/\r\n/g, "\n")}
Standard error:
${result.stderr.toString().replace(/\r\n/g, "\n")}`;
});
});
});
}
}

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,13 @@
Exit Code: 2
Standard output:
node_modules/electron/electron.d.ts(5390,13): error TS2430: Interface 'WebviewTag' incorrectly extends interface 'HTMLElement'.
Types of property 'addEventListener' are incompatible.
Type '{ (event: "load-commit", listener: (event: LoadCommitEvent) => void, useCapture?: boolean | undef...' is not assignable to type '{ <K extends "error" | "waiting" | "progress" | "ended" | "change" | "input" | "select" | "abort"...'.
Types of parameters 'listener' and 'listener' are incompatible.
Type 'EventListenerOrEventListenerObject' is not assignable to type '(event: LoadCommitEvent) => void'.
Type 'EventListenerObject' is not assignable to type '(event: LoadCommitEvent) => void'.
Type 'EventListenerObject' provides no match for the signature '(event: LoadCommitEvent): void'.
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,20 @@
Exit Code: 2
Standard output:
node_modules/abstract-leveldown/index.d.ts(2,3): error TS7010: 'open', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(3,3): error TS7010: 'open', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(5,3): error TS7010: 'close', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(7,3): error TS7010: 'get', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(7,26): error TS7006: Parameter 'err' implicitly has an 'any' type.
node_modules/abstract-leveldown/index.d.ts(8,3): error TS7010: 'get', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(8,39): error TS7006: Parameter 'err' implicitly has an 'any' type.
node_modules/abstract-leveldown/index.d.ts(10,3): error TS7010: 'put', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(11,3): error TS7010: 'put', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(13,3): error TS7010: 'del', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(14,3): error TS7010: 'del', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(17,3): error TS7010: 'batch', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/abstract-leveldown/index.d.ts(18,3): error TS7010: 'batch', which lacks return-type annotation, implicitly has an 'any' return type.
node_modules/leveldown/leveldown.d.ts(66,3): error TS7010: 'seek', which lacks return-type annotation, implicitly has an 'any' return type.
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,12 @@
Exit Code: 2
Standard output:
node_modules/rxjs/scheduler/VirtualTimeScheduler.d.ts(22,22): error TS2415: Class 'VirtualAction<T>' incorrectly extends base class 'AsyncAction<T>'.
Types of property 'work' are incompatible.
Type '(this: VirtualAction<T>, state?: T | undefined) => void' is not assignable to type '(this: AsyncAction<T>, state?: T | undefined) => void'.
The 'this' types of each signature are incompatible.
Type 'AsyncAction<T>' is not assignable to type 'VirtualAction<T>'.
Property 'index' is missing in type 'AsyncAction<T>'.
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,7 @@
Exit Code: 2
Standard output:
node_modules/vuex/types/index.d.ts(124,16): error TS2714: The expression of an export assignment must be an identifier or qualified name in an ambient context.
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1,6 @@
Exit Code: 0
Standard output:
Standard error:

View File

@ -0,0 +1 @@
import ajv = require("ajv");

View File

@ -0,0 +1,11 @@
{
"name": "ajv-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"ajv": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import antd = require("antd");

View File

@ -0,0 +1,12 @@
{
"name": "antd-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"@types/react": "^16.0.18",
"antd": "latest"
}
}

View File

@ -0,0 +1,8 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015", "dom"],
"types": [],
"allowSyntheticDefaultImports": true
}
}

View File

@ -0,0 +1 @@
import axios = require("axios");

View File

@ -0,0 +1,11 @@
{
"name": "axios-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"axios": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import bignumber_js = require("bignumber.js");

View File

@ -0,0 +1,11 @@
{
"name": "bignumber.js-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"bignumber.js": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import discord_js = require("discord.js");

View File

@ -0,0 +1,12 @@
{
"name": "discord.js-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"@types/node": "^8.0.47",
"discord.js": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": ["node"]
}
}

View File

@ -0,0 +1 @@
import electron = require("electron");

View File

@ -0,0 +1,11 @@
{
"name": "electron-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"electron": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015", "dom"],
"types": []
}
}

View File

@ -0,0 +1 @@
import eventemitter2 = require("eventemitter2");

View File

@ -0,0 +1,11 @@
{
"name": "eventemitter2-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"eventemitter2": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import eventemitter3 = require("eventemitter3");

View File

@ -0,0 +1,11 @@
{
"name": "eventemitter3-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"eventemitter3": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import firebase = require("firebase");

View File

@ -0,0 +1,11 @@
{
"name": "firebase-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"firebase": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import github = require("github");

View File

@ -0,0 +1,11 @@
{
"name": "github-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"github": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import immutable = require("immutable");

View File

@ -0,0 +1,11 @@
{
"name": "immutable-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"immutable": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import isobject = require("isobject");

View File

@ -0,0 +1,11 @@
{
"name": "isobject-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"isobject": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import jimp = require("jimp");

View File

@ -0,0 +1,11 @@
{
"name": "jimp-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"jimp": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": ["node"]
}
}

View File

@ -0,0 +1 @@
import jsonschema = require("jsonschema");

View File

@ -0,0 +1,11 @@
{
"name": "jsonschema-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"jsonschema": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import keycode = require("keycode");

View File

@ -0,0 +1,12 @@
{
"name": "keycode-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"@types/node": "^8.0.47",
"keycode": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015", "dom"],
"types": []
}
}

View File

@ -0,0 +1 @@
import leveldown = require("leveldown");

View File

@ -0,0 +1,12 @@
{
"name": "leveldown-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"@types/node": "^8.0.47",
"leveldown": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": ["node"]
}
}

View File

@ -0,0 +1 @@
import localforage = require("localforage");

View File

@ -0,0 +1,11 @@
{
"name": "localforage-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"localforage": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015", "dom"],
"types": []
}
}

View File

@ -0,0 +1 @@
import log4js = require("log4js");

View File

@ -0,0 +1,11 @@
{
"name": "log4js-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"log4js": "latest"
}
}

View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"strict": true,
"lib": ["es2015"],
"types": []
}
}

View File

@ -0,0 +1 @@
import mobx = require("mobx");

View File

@ -0,0 +1,11 @@
{
"name": "mobx-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"license": "Apache-2.0",
"dependencies": {
"mobx": "latest"
}
}

Some files were not shown because too many files have changed in this diff Show More