From 8cec0fc023dd3f04c6de3e057fb49f02c44e16cf Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 11:15:23 -0700 Subject: [PATCH 01/34] Remote resolver (#21332) This is branch will serve as a feature branch for all changes related to switching to the remote resolver. This will include - switching from using the testAdapter to parse the return data to now using this new class resultResolver - adding tests for all testAdapters, fixing for server and adding for resultResolver - moving sendCommand to a new file, out of the server, and getting pytest to adopt it - moving the server which send the test IDs to a new file and adopt it for both pytest and unittest - write tests for these two new files. --- src/client/testing/common/socketServer.ts | 2 +- .../testController/common/resultResolver.ts | 321 +++++++++++ .../testing/testController/common/server.ts | 43 +- .../testing/testController/common/types.ts | 12 +- .../testing/testController/controller.ts | 16 +- .../pytest/pytestDiscoveryAdapter.ts | 73 +-- .../pytest/pytestExecutionAdapter.ts | 49 +- .../unittest/testDiscoveryAdapter.ts | 36 +- .../unittest/testExecutionAdapter.ts | 47 +- .../testController/workspaceTestAdapter.ts | 295 +--------- .../pytestDiscoveryAdapter.unit.test.ts | 123 +++-- .../pytestExecutionAdapter.unit.test.ts | 123 +++-- .../testController/server.unit.test.ts | 507 +++++++++++------ .../testDiscoveryAdapter.unit.test.ts | 62 +-- .../workspaceTestAdapter.unit.test.ts | 513 +++++++++--------- 15 files changed, 1301 insertions(+), 921 deletions(-) create mode 100644 src/client/testing/testController/common/resultResolver.ts diff --git a/src/client/testing/common/socketServer.ts b/src/client/testing/common/socketServer.ts index 554d8c8a0c76..c27bf5a1606c 100644 --- a/src/client/testing/common/socketServer.ts +++ b/src/client/testing/common/socketServer.ts @@ -123,7 +123,7 @@ export class UnitTestSocketServer extends EventEmitter implements IUnitTestSocke if ((socket as any).id) { destroyedSocketId = (socket as any).id; } - this.log('socket disconnected', destroyedSocketId.toString()); + this.log('socket disconnected', destroyedSocketId?.toString()); if (socket && socket.destroy) { socket.destroy(); } diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts new file mode 100644 index 000000000000..8f8cb859b9d5 --- /dev/null +++ b/src/client/testing/testController/common/resultResolver.ts @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { + CancellationToken, + Position, + TestController, + TestItem, + Uri, + Range, + TestMessage, + Location, + TestRun, +} from 'vscode'; +import * as util from 'util'; +import * as path from 'path'; +import { + DiscoveredTestItem, + DiscoveredTestNode, + DiscoveredTestPayload, + ExecutionTestPayload, + ITestResultResolver, +} from './types'; +import { TestProvider } from '../../types'; +import { traceError, traceLog } from '../../../logging'; +import { Testing } from '../../../common/utils/localize'; +import { + DebugTestTag, + ErrorTestItemOptions, + RunTestTag, + clearAllChildren, + createErrorTestItem, + getTestCaseNodes, +} from './testItemUtilities'; +import { sendTelemetryEvent } from '../../../telemetry'; +import { EventName } from '../../../telemetry/constants'; +import { splitLines } from '../../../common/stringUtils'; +import { fixLogLines } from './utils'; + +export class PythonResultResolver implements ITestResultResolver { + testController: TestController; + + testProvider: TestProvider; + + public runIdToTestItem: Map; + + public runIdToVSid: Map; + + public vsIdToRunId: Map; + + constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) { + this.testController = testController; + this.testProvider = testProvider; + + this.runIdToTestItem = new Map(); + this.runIdToVSid = new Map(); + this.vsIdToRunId = new Map(); + } + + public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { + const workspacePath = this.workspaceUri.fsPath; + traceLog('Using result resolver for discovery'); + + const rawTestData = payload; + if (!rawTestData) { + // No test data is available + return Promise.resolve(); + } + + // Check if there were any errors in the discovery process. + if (rawTestData.status === 'error') { + const testingErrorConst = + this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery; + const { errors } = rawTestData; + traceError(testingErrorConst, '\r\n', errors!.join('\r\n\r\n')); + + let errorNode = this.testController.items.get(`DiscoveryError:${workspacePath}`); + const message = util.format( + `${testingErrorConst} ${Testing.seePythonOutput}\r\n`, + errors!.join('\r\n\r\n'), + ); + + if (errorNode === undefined) { + const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider); + errorNode = createErrorTestItem(this.testController, options); + this.testController.items.add(errorNode); + } + errorNode.error = message; + } else { + // Remove the error node if necessary, + // then parse and insert test data. + this.testController.items.delete(`DiscoveryError:${workspacePath}`); + + if (rawTestData.tests) { + // If the test root for this folder exists: Workspace refresh, update its children. + // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree. + populateTestTree(this.testController, rawTestData.tests, undefined, this, token); + } else { + // Delete everything from the test controller. + this.testController.items.replace([]); + } + } + + sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { + tool: this.testProvider, + failed: false, + }); + return Promise.resolve(); + } + + public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { + const rawTestExecData = payload; + if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { + // Map which holds the subtest information for each test item. + const subTestStats: Map = new Map(); + + // iterate through payload and update the UI accordingly. + for (const keyTemp of Object.keys(rawTestExecData.result)) { + const testCases: TestItem[] = []; + + // grab leaf level test items + this.testController.items.forEach((i) => { + const tempArr: TestItem[] = getTestCaseNodes(i); + testCases.push(...tempArr); + }); + + if ( + rawTestExecData.result[keyTemp].outcome === 'failure' || + rawTestExecData.result[keyTemp].outcome === 'passed-unexpected' + ) { + const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? ''; + const traceback = splitLines(rawTraceback, { + trim: false, + removeEmptyEntries: true, + }).join('\r\n'); + + const text = `${rawTestExecData.result[keyTemp].test} failed: ${ + rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome + }\r\n${traceback}\r\n`; + const message = new TestMessage(text); + + // note that keyTemp is a runId for unittest library... + const grabVSid = this.runIdToVSid.get(keyTemp); + // search through freshly built array of testItem to find the failed test and update UI. + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + message.location = new Location(indiItem.uri, indiItem.range); + runInstance.failed(indiItem, message); + runInstance.appendOutput(fixLogLines(text)); + } + } + }); + } else if ( + rawTestExecData.result[keyTemp].outcome === 'success' || + rawTestExecData.result[keyTemp].outcome === 'expected-failure' + ) { + const grabTestItem = this.runIdToTestItem.get(keyTemp); + const grabVSid = this.runIdToVSid.get(keyTemp); + if (grabTestItem !== undefined) { + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + runInstance.passed(grabTestItem); + runInstance.appendOutput('Passed here'); + } + } + }); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'skipped') { + const grabTestItem = this.runIdToTestItem.get(keyTemp); + const grabVSid = this.runIdToVSid.get(keyTemp); + if (grabTestItem !== undefined) { + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + runInstance.skipped(grabTestItem); + runInstance.appendOutput('Skipped here'); + } + } + }); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') { + // split on " " since the subtest ID has the parent test ID in the first part of the ID. + const parentTestCaseId = keyTemp.split(' ')[0]; + const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); + const data = rawTestExecData.result[keyTemp]; + // find the subtest's parent test item + if (parentTestItem) { + const subtestStats = subTestStats.get(parentTestCaseId); + if (subtestStats) { + subtestStats.failed += 1; + } else { + subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); + runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); + // clear since subtest items don't persist between runs + clearAllChildren(parentTestItem); + } + const subtestId = keyTemp; + const subTestItem = this.testController?.createTestItem(subtestId, subtestId); + runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`)); + // create a new test item for the subtest + if (subTestItem) { + const traceback = data.traceback ?? ''; + const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`; + runInstance.appendOutput(fixLogLines(text)); + parentTestItem.children.add(subTestItem); + runInstance.started(subTestItem); + const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? ''); + if (parentTestItem.uri && parentTestItem.range) { + message.location = new Location(parentTestItem.uri, parentTestItem.range); + } + runInstance.failed(subTestItem, message); + } else { + throw new Error('Unable to create new child node for subtest'); + } + } else { + throw new Error('Parent test item not found'); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') { + // split on " " since the subtest ID has the parent test ID in the first part of the ID. + const parentTestCaseId = keyTemp.split(' ')[0]; + const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); + + // find the subtest's parent test item + if (parentTestItem) { + const subtestStats = subTestStats.get(parentTestCaseId); + if (subtestStats) { + subtestStats.passed += 1; + } else { + subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); + runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); + // clear since subtest items don't persist between runs + clearAllChildren(parentTestItem); + } + const subtestId = keyTemp; + const subTestItem = this.testController?.createTestItem(subtestId, subtestId); + // create a new test item for the subtest + if (subTestItem) { + parentTestItem.children.add(subTestItem); + runInstance.started(subTestItem); + runInstance.passed(subTestItem); + runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`)); + } else { + throw new Error('Unable to create new child node for subtest'); + } + } else { + throw new Error('Parent test item not found'); + } + } + } + } + return Promise.resolve(); + } +} +// had to switch the order of the original parameter since required param cannot follow optional. +function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 6bd9bf348e20..32829e355ccb 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -9,7 +9,7 @@ import { IPythonExecutionFactory, SpawnOptions, } from '../../../common/process/types'; -import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; +import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; @@ -24,6 +24,10 @@ export class PythonTestServer implements ITestServer, Disposable { private ready: Promise; + private _onRunDataReceived: EventEmitter = new EventEmitter(); + + private _onDiscoveryDataReceived: EventEmitter = new EventEmitter(); + constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) { this.server = net.createServer((socket: net.Socket) => { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data @@ -48,11 +52,28 @@ export class PythonTestServer implements ITestServer, Disposable { rawData = rpcHeaders.remainingRawData; const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData); const extractedData = rpcContent.extractedJSON; + // do not send until we have the full content if (extractedData.length === Number(totalContentLength)) { - // do not send until we have the full content - traceVerbose(`Received data from test server: ${extractedData}`); - this._onDataReceived.fire({ uuid, data: extractedData }); - this.uuids = this.uuids.filter((u) => u !== uuid); + // if the rawData includes tests then this is a discovery request + if (rawData.includes(`"tests":`)) { + this._onDiscoveryDataReceived.fire({ + uuid, + data: rpcContent.extractedJSON, + }); + // if the rawData includes result then this is a run request + } else if (rawData.includes(`"result":`)) { + this._onRunDataReceived.fire({ + uuid, + data: rpcContent.extractedJSON, + }); + } else { + traceLog( + `Error processing test server request: request is not recognized as discovery or run.`, + ); + this._onDataReceived.fire({ uuid: '', data: '' }); + return; + } + // this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO?? buffer = Buffer.alloc(0); } else { break; @@ -97,6 +118,18 @@ export class PythonTestServer implements ITestServer, Disposable { return uuid; } + public deleteUUID(uuid: string): void { + this.uuids = this.uuids.filter((u) => u !== uuid); + } + + public get onRunDataReceived(): Event { + return this._onRunDataReceived.event; + } + + public get onDiscoveryDataReceived(): Event { + return this._onDiscoveryDataReceived.event; + } + public dispose(): void { this.server.close(); this._onDataReceived.dispose(); diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 4307d7a3913f..cb7fda797c4a 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -172,12 +172,21 @@ export type TestCommandOptionsPytest = { */ export interface ITestServer { readonly onDataReceived: Event; + readonly onRunDataReceived: Event; + readonly onDiscoveryDataReceived: Event; sendCommand(options: TestCommandOptions, runTestIdsPort?: string, callback?: () => void): Promise; serverReady(): Promise; getPort(): number; createUUID(cwd: string): string; + deleteUUID(uuid: string): void; +} +export interface ITestResultResolver { + runIdToVSid: Map; + runIdToTestItem: Map; + vsIdToRunId: Map; + resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; + resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; } - export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature discoverTests(uri: Uri): Promise; @@ -192,6 +201,7 @@ export interface ITestExecutionAdapter { uri: Uri, testIds: string[], debugBool?: boolean, + runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise; diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 0d3487855380..eff333a4cdd9 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -31,12 +31,14 @@ import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../common/constants'; import { TestProvider } from '../types'; import { PythonTestServer } from './common/server'; import { DebugTestTag, getNodeByUri, RunTestTag } from './common/testItemUtilities'; +import { pythonTestAdapterRewriteEnabled } from './common/utils'; import { ITestController, ITestDiscoveryAdapter, ITestFrameworkController, TestRefreshOptions, ITestExecutionAdapter, + ITestResultResolver, } from './common/types'; import { UnittestTestDiscoveryAdapter } from './unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from './unittest/testExecutionAdapter'; @@ -44,8 +46,8 @@ import { PytestTestDiscoveryAdapter } from './pytest/pytestDiscoveryAdapter'; import { PytestTestExecutionAdapter } from './pytest/pytestExecutionAdapter'; import { WorkspaceTestAdapter } from './workspaceTestAdapter'; import { ITestDebugLauncher } from '../common/types'; -import { pythonTestAdapterRewriteEnabled } from './common/utils'; import { IServiceContainer } from '../../ioc/types'; +import { PythonResultResolver } from './common/resultResolver'; // Types gymnastics to make sure that sendTriggerTelemetry only accepts the correct types. type EventPropertyType = IEventNamePropertyMapping[EventName.UNITTEST_DISCOVERY_TRIGGER]; @@ -161,30 +163,37 @@ export class PythonTestController implements ITestController, IExtensionSingleAc let discoveryAdapter: ITestDiscoveryAdapter; let executionAdapter: ITestExecutionAdapter; let testProvider: TestProvider; + let resultResolver: ITestResultResolver; if (settings.testing.unittestEnabled) { + testProvider = UNITTEST_PROVIDER; + resultResolver = new PythonResultResolver(this.testController, testProvider, workspace.uri); discoveryAdapter = new UnittestTestDiscoveryAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); executionAdapter = new UnittestTestExecutionAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); - testProvider = UNITTEST_PROVIDER; } else { + testProvider = PYTEST_PROVIDER; + resultResolver = new PythonResultResolver(this.testController, testProvider, workspace.uri); discoveryAdapter = new PytestTestDiscoveryAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); executionAdapter = new PytestTestExecutionAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); - testProvider = PYTEST_PROVIDER; } const workspaceTestAdapter = new WorkspaceTestAdapter( @@ -192,6 +201,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc discoveryAdapter, executionAdapter, workspace.uri, + resultResolver, ); this.testAdapters.set(workspace.uri, workspaceTestAdapter); diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index aeb920407cd2..4378c68b534c 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -10,8 +10,14 @@ import { import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; -import { traceError, traceLog, traceVerbose } from '../../../logging'; -import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, ITestServer } from '../common/types'; +import { traceError, traceVerbose } from '../../../logging'; +import { + DataReceivedEvent, + DiscoveredTestPayload, + ITestDiscoveryAdapter, + ITestResultResolver, + ITestServer, +} from '../common/types'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. #this seems incorrectly copied @@ -19,39 +25,32 @@ import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, ITestS export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { private promiseMap: Map> = new Map(); - private deferred: Deferred | undefined; - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} - discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { - if (executionFactory !== undefined) { - // ** new version of discover tests. - const settings = this.configSettings.getSettings(uri); - const { pytestArgs } = settings.testing; - traceVerbose(pytestArgs); - return this.runPytestDiscovery(uri, executionFactory); + async discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { + const settings = this.configSettings.getSettings(uri); + const { pytestArgs } = settings.testing; + traceVerbose(pytestArgs); + const disposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { + // cancelation token ? + this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + }); + try { + await this.runPytestDiscovery(uri, executionFactory); + } finally { + disposable.dispose(); } - // if executionFactory is undefined, we are using the old method signature of discover tests. - traceVerbose(uri); - this.deferred = createDeferred(); - return this.deferred.promise; + // this is only a placeholder to handle function overloading until rewrite is finished + const discoveryPayload: DiscoveredTestPayload = { cwd: uri.fsPath, status: 'success' }; + return discoveryPayload; } - async runPytestDiscovery(uri: Uri, executionFactory: IPythonExecutionFactory): Promise { + async runPytestDiscovery(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { const deferred = createDeferred(); const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); @@ -79,13 +78,19 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { allowEnvironmentFetchExceptions: false, resource: uri, }; - const execService = await executionFactory.createActivatedEnvironment(creationOptions); - const discoveryArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs); - traceLog(`Discovering pytest tests with arguments: ${discoveryArgs.join(' ')}`); - execService.exec(discoveryArgs, spawnOptions).catch((ex) => { - traceError(`Error occurred while discovering tests: ${ex}`); - deferred.reject(ex as Error); - }); + const execService = await executionFactory?.createActivatedEnvironment(creationOptions); + // delete UUID following entire discovery finishing. + execService + ?.exec(['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs), spawnOptions) + .then(() => { + this.testServer.deleteUUID(uuid); + return deferred.resolve(); + }) + .catch((err) => { + traceError(`Error while trying to run pytest discovery, \n${err}\r\n\r\n`); + this.testServer.deleteUUID(uuid); + return deferred.reject(err); + }); return deferred.promise; } } diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 90704b5d67f4..6d63b49d007f 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -1,13 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { Uri } from 'vscode'; +import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; -import { DataReceivedEvent, ExecutionTestPayload, ITestExecutionAdapter, ITestServer } from '../common/types'; +import { + DataReceivedEvent, + ExecutionTestPayload, + ITestExecutionAdapter, + ITestResultResolver, + ITestServer, +} from '../common/types'; import { ExecutionFactoryCreateWithEnvironmentOptions, IPythonExecutionFactory, @@ -27,39 +33,35 @@ import { EXTENSION_ROOT_DIR } from '../../../common/constants'; export class PytestTestExecutionAdapter implements ITestExecutionAdapter { private promiseMap: Map> = new Map(); - private deferred: Deferred | undefined; - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} async runTests( uri: Uri, testIds: string[], debugBool?: boolean, + runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { - if (executionFactory !== undefined) { - // ** new version of run tests. - return this.runTestsNew(uri, testIds, debugBool, executionFactory, debugLauncher); + traceVerbose(uri, testIds, debugBool); + const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { + if (runInstance) { + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + } + }); + try { + await this.runTestsNew(uri, testIds, debugBool, executionFactory, debugLauncher); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) } - // if executionFactory is undefined, we are using the old method signature of run tests. - this.outputChannel.appendLine('Running tests.'); - this.deferred = createDeferred(); - return this.deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } private async runTestsNew( @@ -187,6 +189,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { return Promise.reject(ex); } - return deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } } diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 9c565af78c08..8d393a8da18d 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -10,11 +10,11 @@ import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, + ITestResultResolver, ITestServer, TestCommandOptions, TestDiscoveryCommand, } from '../common/types'; -import { traceInfo } from '../../../logging'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. @@ -28,17 +28,8 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} public async discoverTests(uri: Uri): Promise { const deferred = createDeferred(); @@ -60,12 +51,23 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { this.promiseMap.set(uuid, deferred); - // Send the test command to the server. - // The server will fire an onDataReceived event once it gets a response. - traceInfo(`Sending discover unittest script to server.`); - this.testServer.sendCommand(options); + const disposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { + this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + }); + try { + await this.callSendCommand(options); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) + } + const discoveryPayload: DiscoveredTestPayload = { cwd: uri.fsPath, status: 'success' }; + return discoveryPayload; + } - return deferred.promise; + private async callSendCommand(options: TestCommandOptions): Promise { + await this.testServer.sendCommand(options); + const discoveryPayload: DiscoveredTestPayload = { cwd: '', status: 'success' }; + return discoveryPayload; } } diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index bf83c3c0feb1..16e7d03804a1 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as path from 'path'; -import { Uri } from 'vscode'; +import { TestRun, Uri } from 'vscode'; import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; @@ -11,6 +11,7 @@ import { DataReceivedEvent, ExecutionTestPayload, ITestExecutionAdapter, + ITestResultResolver, ITestServer, TestCommandOptions, TestExecutionCommand, @@ -30,19 +31,31 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); + private readonly resultResolver?: ITestResultResolver, + ) {} + + public async runTests( + uri: Uri, + testIds: string[], + debugBool?: boolean, + runInstance?: TestRun, + ): Promise { + const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { + if (runInstance) { + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + } + }); + try { + await this.runTestsNew(uri, testIds, debugBool); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) } + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } - public async runTests(uri: Uri, testIds: string[], debugBool?: boolean): Promise { + private async runTestsNew(uri: Uri, testIds: string[], debugBool?: boolean): Promise { const settings = this.configSettings.getSettings(uri); const { unittestArgs } = settings.testing; @@ -62,7 +75,6 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const deferred = createDeferred(); this.promiseMap.set(uuid, deferred); - // create payload with testIds to send to run pytest script const testData = JSON.stringify(testIds); const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; @@ -99,15 +111,18 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { runTestIdsPort = assignedPort.toString(); // Send test command to server. // Server fire onDataReceived event once it gets response. - this.testServer.sendCommand(options, runTestIdsPort, () => { - deferred.resolve(); - }); }) .catch((error) => { traceError('Error starting server:', error); }); - return deferred.promise; + await this.testServer.sendCommand(options, runTestIdsPort, () => { + // disposable.dispose(); + deferred.resolve(); + }); + // return deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } } diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5cba6c193d3c..5c7ee9cfe520 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -3,34 +3,15 @@ import * as path from 'path'; import * as util from 'util'; -import { - CancellationToken, - Position, - Range, - TestController, - TestItem, - TestMessage, - TestRun, - Uri, - Location, -} from 'vscode'; -import { splitLines } from '../../common/stringUtils'; +import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; import { Testing } from '../../common/utils/localize'; -import { traceError, traceVerbose } from '../../logging'; +import { traceError } from '../../logging'; import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { TestProvider } from '../types'; -import { - clearAllChildren, - createErrorTestItem, - DebugTestTag, - ErrorTestItemOptions, - getTestCaseNodes, - RunTestTag, -} from './common/testItemUtilities'; -import { DiscoveredTestItem, DiscoveredTestNode, ITestDiscoveryAdapter, ITestExecutionAdapter } from './common/types'; -import { fixLogLines } from './common/utils'; +import { createErrorTestItem, ErrorTestItemOptions, getTestCaseNodes } from './common/testItemUtilities'; +import { ITestDiscoveryAdapter, ITestExecutionAdapter, ITestResultResolver } from './common/types'; import { IPythonExecutionFactory } from '../../common/process/types'; import { ITestDebugLauncher } from '../common/types'; @@ -48,22 +29,13 @@ export class WorkspaceTestAdapter { private executing: Deferred | undefined; - runIdToTestItem: Map; - - runIdToVSid: Map; - - vsIdToRunId: Map; - constructor( private testProvider: TestProvider, private discoveryAdapter: ITestDiscoveryAdapter, private executionAdapter: ITestExecutionAdapter, private workspaceUri: Uri, - ) { - this.runIdToTestItem = new Map(); - this.runIdToVSid = new Map(); - this.vsIdToRunId = new Map(); - } + private resultResolver: ITestResultResolver, + ) {} public async executeTests( testController: TestController, @@ -81,7 +53,6 @@ export class WorkspaceTestAdapter { const deferred = createDeferred(); this.executing = deferred; - let rawTestExecData; const testCaseNodes: TestItem[] = []; const testCaseIdsSet = new Set(); try { @@ -93,7 +64,7 @@ export class WorkspaceTestAdapter { // iterate through testItems nodes and fetch their unittest runID to pass in as argument testCaseNodes.forEach((node) => { runInstance.started(node); // do the vscode ui test item start here before runtest - const runId = this.vsIdToRunId.get(node.id); + const runId = this.resultResolver.vsIdToRunId.get(node.id); if (runId) { testCaseIdsSet.add(runId); } @@ -101,16 +72,16 @@ export class WorkspaceTestAdapter { const testCaseIds = Array.from(testCaseIdsSet); // ** execution factory only defined for new rewrite way if (executionFactory !== undefined) { - traceVerbose('executionFactory defined'); - rawTestExecData = await this.executionAdapter.runTests( + await this.executionAdapter.runTests( this.workspaceUri, testCaseIds, debugBool, + runInstance, executionFactory, debugLauncher, ); } else { - rawTestExecData = await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, debugBool); + await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, debugBool); } deferred.resolve(); } catch (ex) { @@ -136,146 +107,6 @@ export class WorkspaceTestAdapter { this.executing = undefined; } - if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { - // Map which holds the subtest information for each test item. - const subTestStats: Map = new Map(); - - // iterate through payload and update the UI accordingly. - for (const keyTemp of Object.keys(rawTestExecData.result)) { - const testCases: TestItem[] = []; - - // grab leaf level test items - testController.items.forEach((i) => { - const tempArr: TestItem[] = getTestCaseNodes(i); - testCases.push(...tempArr); - }); - - if ( - rawTestExecData.result[keyTemp].outcome === 'failure' || - rawTestExecData.result[keyTemp].outcome === 'passed-unexpected' - ) { - const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? ''; - const traceback = splitLines(rawTraceback, { - trim: false, - removeEmptyEntries: true, - }).join('\r\n'); - - const text = `${rawTestExecData.result[keyTemp].test} failed: ${ - rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome - }\r\n${traceback}\r\n`; - const message = new TestMessage(text); - - // note that keyTemp is a runId for unittest library... - const grabVSid = this.runIdToVSid.get(keyTemp); - // search through freshly built array of testItem to find the failed test and update UI. - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - message.location = new Location(indiItem.uri, indiItem.range); - runInstance.failed(indiItem, message); - runInstance.appendOutput(fixLogLines(text)); - } - } - }); - } else if ( - rawTestExecData.result[keyTemp].outcome === 'success' || - rawTestExecData.result[keyTemp].outcome === 'expected-failure' - ) { - const grabTestItem = this.runIdToTestItem.get(keyTemp); - const grabVSid = this.runIdToVSid.get(keyTemp); - if (grabTestItem !== undefined) { - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - runInstance.passed(grabTestItem); - runInstance.appendOutput('Passed here'); - } - } - }); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'skipped') { - const grabTestItem = this.runIdToTestItem.get(keyTemp); - const grabVSid = this.runIdToVSid.get(keyTemp); - if (grabTestItem !== undefined) { - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - runInstance.skipped(grabTestItem); - runInstance.appendOutput('Skipped here'); - } - } - }); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') { - // split on " " since the subtest ID has the parent test ID in the first part of the ID. - const parentTestCaseId = keyTemp.split(' ')[0]; - const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); - const data = rawTestExecData.result[keyTemp]; - // find the subtest's parent test item - if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); - if (subtestStats) { - subtestStats.failed += 1; - } else { - subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); - runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); - // clear since subtest items don't persist between runs - clearAllChildren(parentTestItem); - } - const subtestId = keyTemp; - const subTestItem = testController?.createTestItem(subtestId, subtestId); - runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`)); - // create a new test item for the subtest - if (subTestItem) { - const traceback = data.traceback ?? ''; - const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`; - runInstance.appendOutput(fixLogLines(text)); - parentTestItem.children.add(subTestItem); - runInstance.started(subTestItem); - const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? ''); - if (parentTestItem.uri && parentTestItem.range) { - message.location = new Location(parentTestItem.uri, parentTestItem.range); - } - runInstance.failed(subTestItem, message); - } else { - throw new Error('Unable to create new child node for subtest'); - } - } else { - throw new Error('Parent test item not found'); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') { - // split on " " since the subtest ID has the parent test ID in the first part of the ID. - const parentTestCaseId = keyTemp.split(' ')[0]; - const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); - - // find the subtest's parent test item - if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); - if (subtestStats) { - subtestStats.passed += 1; - } else { - subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); - runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); - // clear since subtest items don't persist between runs - clearAllChildren(parentTestItem); - } - const subtestId = keyTemp; - const subTestItem = testController?.createTestItem(subtestId, subtestId); - // create a new test item for the subtest - if (subTestItem) { - parentTestItem.children.add(subTestItem); - runInstance.started(subTestItem); - runInstance.passed(subTestItem); - runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`)); - } else { - throw new Error('Unable to create new child node for subtest'); - } - } else { - throw new Error('Parent test item not found'); - } - } - } - } return Promise.resolve(); } @@ -286,8 +117,6 @@ export class WorkspaceTestAdapter { ): Promise { sendTelemetryEvent(EventName.UNITTEST_DISCOVERING, undefined, { tool: this.testProvider }); - const workspacePath = this.workspaceUri.fsPath; - // Discovery is expensive. If it is already running, use the existing promise. if (this.discovering) { return this.discovering.promise; @@ -296,14 +125,12 @@ export class WorkspaceTestAdapter { const deferred = createDeferred(); this.discovering = deferred; - let rawTestData; try { // ** execution factory only defined for new rewrite way if (executionFactory !== undefined) { - traceVerbose('executionFactory defined'); - rawTestData = await this.discoveryAdapter.discoverTests(this.workspaceUri, executionFactory); + await this.discoveryAdapter.discoverTests(this.workspaceUri, executionFactory); } else { - rawTestData = await this.discoveryAdapter.discoverTests(this.workspaceUri); + await this.discoveryAdapter.discoverTests(this.workspaceUri); } deferred.resolve(); } catch (ex) { @@ -324,114 +151,18 @@ export class WorkspaceTestAdapter { const errorNode = createErrorTestItem(testController, options); testController.items.add(errorNode); - deferred.reject(ex as Error); + return deferred.reject(ex as Error); } finally { // Discovery has finished running, we have the data, // we don't need the deferred promise anymore. this.discovering = undefined; } - if (!rawTestData) { - // No test data is available - return Promise.resolve(); - } - - // Check if there were any errors in the discovery process. - if (rawTestData.status === 'error') { - const testingErrorConst = - this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery; - const { errors } = rawTestData; - traceError(testingErrorConst, '\r\n', errors?.join('\r\n\r\n')); - let errorNode = testController.items.get(`DiscoveryError:${workspacePath}`); - const message = util.format( - `${testingErrorConst} ${Testing.seePythonOutput}\r\n`, - errors?.join('\r\n\r\n'), - ); - - if (errorNode === undefined) { - const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider); - errorNode = createErrorTestItem(testController, options); - testController.items.add(errorNode); - } - errorNode.error = message; - } else { - // Remove the error node if necessary, - // then parse and insert test data. - testController.items.delete(`DiscoveryError:${workspacePath}`); - - if (rawTestData.tests) { - // If the test root for this folder exists: Workspace refresh, update its children. - // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree. - populateTestTree(testController, rawTestData.tests, undefined, this, token); - } else { - // Delete everything from the test controller. - testController.items.replace([]); - } - } - sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: this.testProvider, failed: false }); return Promise.resolve(); } } -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} - -// had to switch the order of the original parameter since required param cannot follow optional. -function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - wstAdapter: WorkspaceTestAdapter, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - wstAdapter.runIdToTestItem.set(child.runID, testItem); - wstAdapter.runIdToVSid.set(child.runID, child.id_); - wstAdapter.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, wstAdapter, token); - } - } - }); -} - function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; return { diff --git a/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts index 12c79a23c7fd..0286235be1bf 100644 --- a/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts @@ -4,11 +4,17 @@ import * as assert from 'assert'; import { Uri } from 'vscode'; import * as typeMoq from 'typemoq'; +import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; import { PytestTestDiscoveryAdapter } from '../../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { DataReceivedEvent, ITestServer } from '../../../../client/testing/testController/common/types'; -import { IPythonExecutionFactory, IPythonExecutionService } from '../../../../client/common/process/types'; +import { ITestServer } from '../../../../client/testing/testController/common/types'; +import { + IPythonExecutionFactory, + IPythonExecutionService, + SpawnOptions, +} from '../../../../client/common/process/types'; import { createDeferred, Deferred } from '../../../../client/common/utils/async'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; suite('pytest test discovery adapter', () => { let testServer: typeMoq.IMock; @@ -18,27 +24,56 @@ suite('pytest test discovery adapter', () => { let execService: typeMoq.IMock; let deferred: Deferred; let outputChannel: typeMoq.IMock; + let portNum: number; + let uuid: string; + let expectedPath: string; + let uri: Uri; + let expectedExtraVariables: Record; setup(() => { + const mockExtensionRootDir = typeMoq.Mock.ofType(); + mockExtensionRootDir.setup((m) => m.toString()).returns(() => '/mocked/extension/root/dir'); + + // constants + portNum = 12345; + uuid = 'uuid123'; + expectedPath = path.join('/', 'my', 'test', 'path'); + uri = Uri.file(expectedPath); + const relativePathToPytest = 'pythonFiles'; + const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); + expectedExtraVariables = { + PYTHONPATH: fullPluginPath, + TEST_UUID: uuid, + TEST_PORT: portNum.toString(), + }; + + // set up test server testServer = typeMoq.Mock.ofType(); - testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer.setup((t) => t.getPort()).returns(() => portNum); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); testServer - .setup((t) => t.onDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns(() => ({ dispose: () => { /* no-body */ }, })); + + // set up config service configService = ({ getSettings: () => ({ testing: { pytestArgs: ['.'] }, }), } as unknown) as IConfigurationService; + + // set up exec factory execFactory = typeMoq.Mock.ofType(); - execService = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) .returns(() => Promise.resolve(execService.object)); + + // set up exec service + execService = typeMoq.Mock.ofType(); deferred = createDeferred(); execService .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) @@ -46,49 +81,51 @@ suite('pytest test discovery adapter', () => { deferred.resolve(); return Promise.resolve({ stdout: '{}' }); }); - execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); outputChannel = typeMoq.Mock.ofType(); }); - test('onDataReceivedHandler should parse only if known UUID', async () => { - const uri = Uri.file('/my/test/path/'); - const uuid = 'uuid123'; - const data = { status: 'success' }; - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); - const eventData: DataReceivedEvent = { - uuid, - data: JSON.stringify(data), - }; - + test('Discovery should call exec with correct basic args', async () => { adapter = new PytestTestDiscoveryAdapter(testServer.object, configService, outputChannel.object); - const promise = adapter.discoverTests(uri, execFactory.object); - // const promise = adapter.discoverTests(uri); - await deferred.promise; - adapter.onDataReceivedHandler(eventData); - const result = await promise; - assert.deepStrictEqual(result, data); + await adapter.discoverTests(uri, execFactory.object); + const expectedArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only', '.']; + + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.deepEqual(options.extraVariables, expectedExtraVariables); + assert.equal(options.cwd, expectedPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); }); - test('onDataReceivedHandler should not parse if it is unknown UUID', async () => { - const uri = Uri.file('/my/test/path/'); - const uuid = 'uuid456'; - let data = { status: 'error' }; - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); - const wrongUriEventData: DataReceivedEvent = { - uuid: 'incorrect-uuid456', - data: JSON.stringify(data), - }; - adapter = new PytestTestDiscoveryAdapter(testServer.object, configService, outputChannel.object); - const promise = adapter.discoverTests(uri, execFactory.object); - // const promise = adapter.discoverTests(uri); - adapter.onDataReceivedHandler(wrongUriEventData); + test('Test discovery correctly pulls pytest args from config service settings', async () => { + // set up a config service with different pytest args + const configServiceNew: IConfigurationService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.', 'abc', 'xyz'] }, + }), + } as unknown) as IConfigurationService; - data = { status: 'success' }; - const correctUriEventData: DataReceivedEvent = { - uuid, - data: JSON.stringify(data), - }; - adapter.onDataReceivedHandler(correctUriEventData); - const result = await promise; - assert.deepStrictEqual(result, data); + adapter = new PytestTestDiscoveryAdapter(testServer.object, configServiceNew, outputChannel.object); + await adapter.discoverTests(uri, execFactory.object); + const expectedArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only', '.', 'abc', 'xyz']; + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.deepEqual(options.extraVariables, expectedExtraVariables); + assert.equal(options.cwd, expectedPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); }); }); diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index ac6c6bd274a4..5e7a32ddc9e1 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -2,13 +2,18 @@ // // Copyright (c) Microsoft Corporation. All rights reserved. // // Licensed under the MIT License. // import * as assert from 'assert'; -// import { Uri } from 'vscode'; +// import { TestRun, Uri } from 'vscode'; // import * as typeMoq from 'typemoq'; -// import { IConfigurationService } from '../../../../client/common/types'; -// import { DataReceivedEvent, ITestServer } from '../../../../client/testing/testController/common/types'; -// import { IPythonExecutionFactory, IPythonExecutionService } from '../../../../client/common/process/types'; +// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +// import { ITestServer } from '../../../../client/testing/testController/common/types'; +// import { +// IPythonExecutionFactory, +// IPythonExecutionService, +// SpawnOptions, +// } from '../../../../client/common/process/types'; // import { createDeferred, Deferred } from '../../../../client/common/utils/async'; // import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; +// import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; // suite('pytest test execution adapter', () => { // let testServer: typeMoq.IMock; @@ -17,11 +22,12 @@ // let adapter: PytestTestExecutionAdapter; // let execService: typeMoq.IMock; // let deferred: Deferred; +// let debugLauncher: typeMoq.IMock; // setup(() => { // testServer = typeMoq.Mock.ofType(); // testServer.setup((t) => t.getPort()).returns(() => 12345); // testServer -// .setup((t) => t.onDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) // .returns(() => ({ // dispose: () => { // /* no-body */ @@ -35,6 +41,7 @@ // } as unknown) as IConfigurationService; // execFactory = typeMoq.Mock.ofType(); // execService = typeMoq.Mock.ofType(); +// debugLauncher = typeMoq.Mock.ofType(); // execFactory // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) // .returns(() => Promise.resolve(execService.object)); @@ -45,46 +52,90 @@ // deferred.resolve(); // return Promise.resolve({ stdout: '{}' }); // }); +// debugLauncher +// .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => { +// deferred.resolve(); +// return Promise.resolve(); +// }); // execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); // execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); +// debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); // }); -// test('onDataReceivedHandler should parse only if known UUID', async () => { +// test('pytest execution called with correct args', async () => { // const uri = Uri.file('/my/test/path/'); // const uuid = 'uuid123'; -// const data = { status: 'success' }; +// // const data = { status: 'success' }; +// testServer +// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => ({ +// dispose: () => { +// /* no-body */ +// }, +// })); // testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const eventData: DataReceivedEvent = { -// uuid, -// data: JSON.stringify(data), -// }; +// const outputChannel = typeMoq.Mock.ofType(); +// const testRun = typeMoq.Mock.ofType(); +// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); +// await adapter.runTests(uri, [], false, testRun.object, execFactory.object); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService); -// const promise = adapter.runTests(uri, [], false); -// await deferred.promise; -// adapter.onDataReceivedHandler(eventData); -// const result = await promise; -// assert.deepStrictEqual(result, data); +// const expectedArgs = [ +// '/Users/eleanorboyd/vscode-python/pythonFiles/vscode_pytest/run_pytest_script.py', +// '--rootdir', +// '/my/test/path/', +// ]; +// const expectedExtraVariables = { +// PYTHONPATH: '/Users/eleanorboyd/vscode-python/pythonFiles', +// TEST_UUID: 'uuid123', +// TEST_PORT: '12345', +// }; +// execService.verify( +// (x) => +// x.exec( +// expectedArgs, +// typeMoq.It.is((options) => { +// assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); +// assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); +// assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); +// assert.strictEqual(typeof options.extraVariables?.RUN_TEST_IDS_PORT, 'string'); +// assert.equal(options.cwd, uri.fsPath); +// assert.equal(options.throwOnStdErr, true); +// return true; +// }), +// ), +// typeMoq.Times.once(), +// ); // }); -// test('onDataReceivedHandler should not parse if it is unknown UUID', async () => { +// test('Debug launched correctly for pytest', async () => { // const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid456'; -// let data = { status: 'error' }; +// const uuid = 'uuid123'; +// testServer +// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => ({ +// dispose: () => { +// /* no-body */ +// }, +// })); // testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const wrongUriEventData: DataReceivedEvent = { -// uuid: 'incorrect-uuid456', -// data: JSON.stringify(data), -// }; -// adapter = new PytestTestExecutionAdapter(testServer.object, configService); -// const promise = adapter.runTests(uri, [], false); -// adapter.onDataReceivedHandler(wrongUriEventData); - -// data = { status: 'success' }; -// const correctUriEventData: DataReceivedEvent = { -// uuid, -// data: JSON.stringify(data), -// }; -// adapter.onDataReceivedHandler(correctUriEventData); -// const result = await promise; -// assert.deepStrictEqual(result, data); +// const outputChannel = typeMoq.Mock.ofType(); +// const testRun = typeMoq.Mock.ofType(); +// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); +// await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); +// debugLauncher.verify( +// (x) => +// x.launchDebugger( +// typeMoq.It.is((launchOptions) => { +// assert.equal(launchOptions.cwd, uri.fsPath); +// assert.deepEqual(launchOptions.args, ['--rootdir', '/my/test/path/', '--capture', 'no']); +// assert.equal(launchOptions.testProvider, 'pytest'); +// assert.equal(launchOptions.pytestPort, '12345'); +// assert.equal(launchOptions.pytestUUID, 'uuid123'); +// assert.strictEqual(typeof launchOptions.runTestIdsPort, 'string'); +// return true; +// }), +// typeMoq.It.isAny(), +// ), +// typeMoq.Times.once(), +// ); // }); // }); diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index d7b3a242ee9a..38b71992aefb 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -1,158 +1,349 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import * as assert from 'assert'; -import * as net from 'net'; -import * as sinon from 'sinon'; -import * as crypto from 'crypto'; -import { OutputChannel, Uri } from 'vscode'; -import { IPythonExecutionFactory, IPythonExecutionService } from '../../../client/common/process/types'; -import { PythonTestServer } from '../../../client/testing/testController/common/server'; -import { ITestDebugLauncher } from '../../../client/testing/common/types'; -import { createDeferred } from '../../../client/common/utils/async'; - -suite('Python Test Server', () => { - const fakeUuid = 'fake-uuid'; - - let stubExecutionFactory: IPythonExecutionFactory; - let stubExecutionService: IPythonExecutionService; - let server: PythonTestServer; - let sandbox: sinon.SinonSandbox; - let execArgs: string[]; - let v4Stub: sinon.SinonStub; - let debugLauncher: ITestDebugLauncher; - - setup(() => { - sandbox = sinon.createSandbox(); - v4Stub = sandbox.stub(crypto, 'randomUUID'); - - v4Stub.returns(fakeUuid); - stubExecutionService = ({ - exec: (args: string[]) => { - execArgs = args; - return Promise.resolve({ stdout: '', stderr: '' }); - }, - } as unknown) as IPythonExecutionService; - - stubExecutionFactory = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService), - } as unknown) as IPythonExecutionFactory; - }); - - teardown(() => { - sandbox.restore(); - execArgs = []; - server.dispose(); - }); - - test('sendCommand should add the port to the command being sent', async () => { - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - await server.sendCommand(options); - const port = server.getPort(); - - assert.deepStrictEqual(execArgs, ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']); - }); - - test('sendCommand should write to an output channel if it is provided as an option', async () => { - const output: string[] = []; - const outChannel = { - appendLine: (str: string) => { - output.push(str); - }, - } as OutputChannel; - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - outChannel, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - await server.sendCommand(options); - - const port = server.getPort(); - const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); - - assert.deepStrictEqual(output, [expected]); - }); - - test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { - let eventData: { status: string; errors: string[] }; - stubExecutionService = ({ - exec: () => { - throw new Error('Failed to execute'); - }, - } as unknown) as IPythonExecutionService; - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - server.onDataReceived(({ data }) => { - eventData = JSON.parse(data); - }); - - await server.sendCommand(options); - - assert.deepStrictEqual(eventData!.status, 'error'); - assert.deepStrictEqual(eventData!.errors, ['Failed to execute']); - }); - - test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const deferred = createDeferred(); - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - stubExecutionService = ({ - exec: async () => { - client.connect(server.getPort()); - return Promise.resolve({ stdout: '', stderr: '' }); - }, - } as unknown) as IPythonExecutionService; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('malformed data'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - await server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); -}); +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. + +// import * as assert from 'assert'; +// import * as net from 'net'; +// import * as sinon from 'sinon'; +// import * as crypto from 'crypto'; +// import { Uri } from 'vscode'; +// import { IPythonExecutionFactory, IPythonExecutionService } from '../../../client/common/process/types'; +// import { PythonTestServer } from '../../../client/testing/testController/common/server'; +// import { ITestDebugLauncher } from '../../../client/testing/common/types'; +// import { createDeferred } from '../../../client/common/utils/async'; + +// suite('Python Test Server', () => { +// const fakeUuid = 'fake-uuid'; + +// let stubExecutionFactory: IPythonExecutionFactory; +// let stubExecutionService: IPythonExecutionService; +// let server: PythonTestServer; +// let sandbox: sinon.SinonSandbox; +// let execArgs: string[]; +// let v4Stub: sinon.SinonStub; +// let debugLauncher: ITestDebugLauncher; + +// setup(() => { +// sandbox = sinon.createSandbox(); +// v4Stub = sandbox.stub(crypto, 'randomUUID'); + +// v4Stub.returns(fakeUuid); +// stubExecutionService = ({ +// exec: (args: string[]) => { +// execArgs = args; +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// stubExecutionFactory = ({ +// createActivatedEnvironment: () => Promise.resolve(stubExecutionService), +// } as unknown) as IPythonExecutionFactory; +// }); + +// teardown(() => { +// sandbox.restore(); +// execArgs = []; +// server.dispose(); +// }); + +// // test('sendCommand should add the port to the command being sent', async () => { +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // await server.sendCommand(options); +// // const port = server.getPort(); + +// // assert.deepStrictEqual(execArgs, ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']); +// // }); + +// // test('sendCommand should write to an output channel if it is provided as an option', async () => { +// // const output: string[] = []; +// // const outChannel = { +// // appendLine: (str: string) => { +// // output.push(str); +// // }, +// // } as OutputChannel; +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // outChannel, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // await server.sendCommand(options); + +// // const port = server.getPort(); +// // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); + +// // assert.deepStrictEqual(output, [expected]); +// // }); + +// // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { +// // let eventData: { status: string; errors: string[] }; +// // stubExecutionService = ({ +// // exec: () => { +// // throw new Error('Failed to execute'); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // server.onDataReceived(({ data }) => { +// // eventData = JSON.parse(data); +// // }); + +// // await server.sendCommand(options); + +// // assert.deepStrictEqual(eventData!.status, 'error'); +// // assert.deepStrictEqual(eventData!.errors, ['Failed to execute']); +// // }); + +// // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('malformed data'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // test('If the server doesnt recognize the UUID it should ignore it', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"Request-uuid": "unknown-uuid"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // required to have "tests" or "results" +// // the heading length not being equal and yes being equal +// // multiple payloads +// // test('Error if payload does not have a content length header', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"not content length": "5"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// const testData = [ +// { +// testName: 'fires discovery correctly on test payload', +// payload: `Content-Length: 52 +// Content-Type: application/json +// Request-uuid: UUID_HERE + +// {"cwd": "path", "status": "success", "tests": "xyz"}`, +// expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', +// }, +// // Add more test data as needed +// ]; + +// testData.forEach(({ testName, payload, expectedResult }) => { +// test(`test: ${testName}`, async () => { +// // Your test logic here +// let eventData: string | undefined; +// const client = new net.Socket(); +// const deferred = createDeferred(); + +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid: fakeUuid, +// }; + +// stubExecutionService = ({ +// exec: async () => { +// client.connect(server.getPort()); +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// await server.serverReady(); +// const uuid = server.createUUID(); +// payload = payload.replace('UUID_HERE', uuid); +// server.onDiscoveryDataReceived(({ data }) => { +// eventData = data; +// deferred.resolve(); +// }); + +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// client.write(payload); +// client.end(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// await server.sendCommand(options); +// await deferred.promise; +// assert.deepStrictEqual(eventData, expectedResult); +// }); +// }); + +// test('Calls run resolver if the result header is in the payload', async () => { +// let eventData: string | undefined; +// const client = new net.Socket(); +// const deferred = createDeferred(); + +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid: fakeUuid, +// }; + +// stubExecutionService = ({ +// exec: async () => { +// client.connect(server.getPort()); +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// await server.serverReady(); +// const uuid = server.createUUID(); +// server.onRunDataReceived(({ data }) => { +// eventData = data; +// deferred.resolve(); +// }); + +// const payload = `Content-Length: 87 +// Content-Type: application/json +// Request-uuid: ${uuid} + +// {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// client.write(payload); +// client.end(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// await server.sendCommand(options); +// await deferred.promise; +// console.log('event data', eventData); +// const expectedResult = +// '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; +// assert.deepStrictEqual(eventData, expectedResult); +// }); +// }); diff --git a/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts index 3d3521291f74..ef21655e93e4 100644 --- a/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts @@ -23,7 +23,7 @@ suite('Unittest test discovery adapter', () => { outputChannel = typemoq.Mock.ofType(); }); - test('discoverTests should send the discovery command to the test server', async () => { + test('DiscoverTests should send the discovery command to the test server with the correct args', async () => { let options: TestCommandOptions | undefined; const stubTestServer = ({ @@ -32,7 +32,7 @@ suite('Unittest test discovery adapter', () => { options = opt; return Promise.resolve(); }, - onDataReceived: () => { + onDiscoveryDataReceived: () => { // no body }, createUUID: () => '123456789', @@ -47,61 +47,11 @@ suite('Unittest test discovery adapter', () => { assert.deepStrictEqual(options, { workspaceFolder: uri, cwd: uri.fsPath, - command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, + command: { + script, + args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'], + }, uuid: '123456789', }); }); - - test("onDataReceivedHandler should parse the data if the cwd from the payload matches the test adapter's cwd", async () => { - const stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - createUUID: () => '123456789', - } as unknown) as ITestServer; - - const uri = Uri.file('/foo/bar'); - const data = { status: 'success' }; - const uuid = '123456789'; - const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - const promise = adapter.discoverTests(uri); - - adapter.onDataReceivedHandler({ uuid, data: JSON.stringify(data) }); - - const result = await promise; - - assert.deepStrictEqual(result, data); - }); - - test("onDataReceivedHandler should ignore the data if the cwd from the payload does not match the test adapter's cwd", async () => { - const correctUuid = '123456789'; - const incorrectUuid = '987654321'; - const stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - createUUID: () => correctUuid, - } as unknown) as ITestServer; - - const uri = Uri.file('/foo/bar'); - - const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - const promise = adapter.discoverTests(uri); - - const data = { status: 'success' }; - adapter.onDataReceivedHandler({ uuid: incorrectUuid, data: JSON.stringify(data) }); - - const nextData = { status: 'error' }; - adapter.onDataReceivedHandler({ uuid: correctUuid, data: JSON.stringify(nextData) }); - - const result = await promise; - - assert.deepStrictEqual(result, nextData); - }); }); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 539647aece9f..42e38d200546 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,246 +1,267 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import * as assert from 'assert'; -import * as sinon from 'sinon'; -import * as typemoq from 'typemoq'; - -import { TestController, TestItem, Uri } from 'vscode'; -import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; -import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 -import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; -import * as Telemetry from '../../../client/telemetry'; -import { EventName } from '../../../client/telemetry/constants'; -import { ITestServer } from '../../../client/testing/testController/common/types'; - -suite('Workspace test adapter', () => { - suite('Test discovery', () => { - let stubTestServer: ITestServer; - let stubConfigSettings: IConfigurationService; - - let discoverTestsStub: sinon.SinonStub; - let sendTelemetryStub: sinon.SinonStub; - let outputChannel: typemoq.IMock; - - let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - - // Stubbed test controller (see comment around L.40) - let testController: TestController; - let log: string[] = []; - - const sandbox = sinon.createSandbox(); - - setup(() => { - stubConfigSettings = ({ - getSettings: () => ({ - testing: { unittestArgs: ['--foo'] }, - }), - } as unknown) as IConfigurationService; - - stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - } as unknown) as ITestServer; - - // For some reason the 'tests' namespace in vscode returns undefined. - // While I figure out how to expose to the tests, they will run - // against a stub test controller and stub test items. - const testItem = ({ - canResolveChildren: false, - tags: [], - children: { - add: () => { - // empty - }, - }, - } as unknown) as TestItem; - - testController = ({ - items: { - get: () => { - log.push('get'); - }, - add: () => { - log.push('add'); - }, - replace: () => { - log.push('replace'); - }, - delete: () => { - log.push('delete'); - }, - }, - createTestItem: () => { - log.push('createTestItem'); - return testItem; - }, - dispose: () => { - // empty - }, - } as unknown) as TestController; - - // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - - const mockSendTelemetryEvent = ( - eventName: EventName, - _: number | Record | undefined, - properties: unknown, - ) => { - telemetryEvent.push({ - eventName, - properties: properties as Record, - }); - }; - - discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); - sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); - outputChannel = typemoq.Mock.ofType(); - }); - - teardown(() => { - telemetryEvent = []; - log = []; - testController.dispose(); - sandbox.restore(); - }); - - test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { - discoverTestsStub.resolves(); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { - discoverTestsStub.callsFake( - async () => - new Promise((resolve) => { - setTimeout(() => { - // Simulate time taken by discovery. - resolve(); - }, 2000); - }), - ); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - // Try running discovery twice - const one = workspaceTestAdapter.discoverTests(testController); - const two = workspaceTestAdapter.discoverTests(testController); - - Promise.all([one, two]); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { - discoverTestsStub.resolves({ status: 'success' }); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); - - const lastEvent = telemetryEvent[1]; - assert.strictEqual(lastEvent.properties.failed, false); - }); - - test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { - discoverTestsStub.rejects(new Error('foo')); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); - - const lastEvent = telemetryEvent[1]; - assert.ok(lastEvent.properties.failed); - - assert.deepStrictEqual(log, ['createTestItem', 'add']); - }); - - /** - * TODO To test: - * - successful discovery but no data: delete everything from the test controller - * - successful discovery with error status: add error node to tree - * - single root: populate tree if there's no root node - * - single root: update tree if there's a root node - * - single root: delete tree if there are no tests in the test data - * - multiroot: update the correct folders - */ - }); -}); +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. + +// import * as assert from 'assert'; +// import * as sinon from 'sinon'; +// import * as typemoq from 'typemoq'; + +// import { TestController, TestItem, Uri } from 'vscode'; +// import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 +// import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +// import * as Telemetry from '../../../client/telemetry'; +// import { EventName } from '../../../client/telemetry/constants'; +// import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; + +// suite('Workspace test adapter', () => { +// suite('Test discovery', () => { +// let stubTestServer: ITestServer; +// let stubConfigSettings: IConfigurationService; +// let stubResultResolver: ITestResultResolver; + +// let discoverTestsStub: sinon.SinonStub; +// let sendTelemetryStub: sinon.SinonStub; +// let outputChannel: typemoq.IMock; + +// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + +// // Stubbed test controller (see comment around L.40) +// let testController: TestController; +// let log: string[] = []; + +// const sandbox = sinon.createSandbox(); + +// setup(() => { +// stubConfigSettings = ({ +// getSettings: () => ({ +// testing: { unittestArgs: ['--foo'] }, +// }), +// } as unknown) as IConfigurationService; + +// stubTestServer = ({ +// sendCommand(): Promise { +// return Promise.resolve(); +// }, +// onDataReceived: () => { +// // no body +// }, +// } as unknown) as ITestServer; + +// stubResultResolver = ({ +// resolveDiscovery: () => { +// // no body +// }, +// resolveExecution: () => { +// // no body +// }, +// vsIdToRunId: { +// get: sinon.stub().returns('expectedRunId'), +// }, +// } as unknown) as ITestResultResolver; + +// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); +// // const expectedRunId = 'expectedRunId'; +// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + +// // For some reason the 'tests' namespace in vscode returns undefined. +// // While I figure out how to expose to the tests, they will run +// // against a stub test controller and stub test items. +// const testItem = ({ +// canResolveChildren: false, +// tags: [], +// children: { +// add: () => { +// // empty +// }, +// }, +// } as unknown) as TestItem; + +// testController = ({ +// items: { +// get: () => { +// log.push('get'); +// }, +// add: () => { +// log.push('add'); +// }, +// replace: () => { +// log.push('replace'); +// }, +// delete: () => { +// log.push('delete'); +// }, +// }, +// createTestItem: () => { +// log.push('createTestItem'); +// return testItem; +// }, +// dispose: () => { +// // empty +// }, +// } as unknown) as TestController; + +// // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + +// const mockSendTelemetryEvent = ( +// eventName: EventName, +// _: number | Record | undefined, +// properties: unknown, +// ) => { +// telemetryEvent.push({ +// eventName, +// properties: properties as Record, +// }); +// }; + +// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); +// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); +// outputChannel = typemoq.Mock.ofType(); +// }); + +// teardown(() => { +// telemetryEvent = []; +// log = []; +// testController.dispose(); +// sandbox.restore(); +// }); + +// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { +// discoverTestsStub.resolves(); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledOnce(discoverTestsStub); +// }); + +// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { +// discoverTestsStub.callsFake( +// async () => +// new Promise((resolve) => { +// setTimeout(() => { +// // Simulate time taken by discovery. +// resolve(); +// }, 2000); +// }), +// ); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// // Try running discovery twice +// const one = workspaceTestAdapter.discoverTests(testController); +// const two = workspaceTestAdapter.discoverTests(testController); + +// Promise.all([one, two]); + +// sinon.assert.calledOnce(discoverTestsStub); +// }); + +// test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { +// discoverTestsStub.resolves({ status: 'success' }); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); + +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); +// assert.strictEqual(telemetryEvent.length, 2); + +// const lastEvent = telemetryEvent[1]; +// assert.strictEqual(lastEvent.properties.failed, false); +// }); + +// test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { +// discoverTestsStub.rejects(new Error('foo')); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); + +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); +// assert.strictEqual(telemetryEvent.length, 2); + +// const lastEvent = telemetryEvent[1]; +// assert.ok(lastEvent.properties.failed); + +// assert.deepStrictEqual(log, ['createTestItem', 'add']); +// }); + +// /** +// * TODO To test: +// * - successful discovery but no data: delete everything from the test controller +// * - successful discovery with error status: add error node to tree +// * - single root: populate tree if there's no root node +// * - single root: update tree if there's a root node +// * - single root: delete tree if there are no tests in the test data +// * - multiroot: update the correct folders +// */ +// }); +// }); From dc5fa4e686a3e6bcd0e82d79f1f95d9c4a8417aa Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 19:00:03 -0700 Subject: [PATCH 02/34] Move testid server (#21400) Created a testIdServer to handle the starting up of the testIdServer for both unittest and pytest. This extracts the method to be in another file and easy to test. --- .../vscode_pytest/run_pytest_script.py | 1 + .../testing/testController/common/utils.ts | 2 +- .../pytest/pytestExecutionAdapter.ts | 36 ++----------------- .../unittest/testExecutionAdapter.ts | 35 +++--------------- 4 files changed, 9 insertions(+), 65 deletions(-) diff --git a/pythonFiles/vscode_pytest/run_pytest_script.py b/pythonFiles/vscode_pytest/run_pytest_script.py index 57bb41a3a7dd..ffb4d0c55b16 100644 --- a/pythonFiles/vscode_pytest/run_pytest_script.py +++ b/pythonFiles/vscode_pytest/run_pytest_script.py @@ -21,6 +21,7 @@ # Add the root directory to the path so that we can import the plugin. directory_path = pathlib.Path(__file__).parent.parent sys.path.append(os.fspath(directory_path)) + sys.path.insert(0, os.getcwd()) # Get the rest of the args to run with pytest. args = sys.argv[1:] run_test_ids_port = os.environ.get("RUN_TEST_IDS_PORT") diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 1bf31e80e11a..9d426dc60ffa 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -62,7 +62,7 @@ export function pythonTestAdapterRewriteEnabled(serviceContainer: IServiceContai return experiment.inExperimentSync(EnableTestAdapterRewrite.experiment); } -export const startServer = (testIds: string): Promise => +export const startTestIdServer = (testIds: string[]): Promise => new Promise((resolve, reject) => { const server = net.createServer((socket: net.Socket) => { // Convert the test_ids array to JSON diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 6d63b49d007f..9c8a42ae097f 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -3,7 +3,6 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; -import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; @@ -23,6 +22,7 @@ import { removePositionalFoldersAndFiles } from './arguments'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { PYTEST_PROVIDER } from '../../common/constants'; import { EXTENSION_ROOT_DIR } from '../../../common/constants'; +import { startTestIdServer } from '../common/utils'; // eslint-disable-next-line @typescript-eslint/no-explicit-any // (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; @@ -116,40 +116,10 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { if (debugBool && !testArgs.some((a) => a.startsWith('--capture') || a === '-s')) { testArgs.push('--capture', 'no'); } - - // create payload with testIds to send to run pytest script - const testData = JSON.stringify(testIds); - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; - traceLog(`Running pytest execution for the following test ids: ${testIds}`); + traceLog(`Running PYTEST execution for the following test ids: ${testIds}`); let pytestRunTestIdsPort: string | undefined; - const startServer = (): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - socket.on('end', () => { - traceVerbose('Client disconnected for pytest test ids server'); - }); - }); - - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceVerbose(`Server listening on port ${port} for pytest test ids server`); - resolve(port); - }); - - server.on('error', (error: Error) => { - traceError('Error starting server for pytest test ids server:', error); - reject(error); - }); - server.on('connection', (socket: net.Socket) => { - socket.write(payload); - traceVerbose('payload sent for pytest execution', payload); - }); - }); - - // Start the server and wait until it is listening - await startServer() + await startTestIdServer(testIds) .then((assignedPort) => { traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); pytestRunTestIdsPort = assignedPort.toString(); diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index 16e7d03804a1..a85490c30bc8 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -3,9 +3,8 @@ import * as path from 'path'; import { TestRun, Uri } from 'vscode'; -import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred, Deferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -17,6 +16,7 @@ import { TestExecutionCommand, } from '../common/types'; import { traceLog, traceError } from '../../../logging'; +import { startTestIdServer } from '../common/utils'; /** * Wrapper Class for unittest test execution. This is where we call `runTestCommand`? @@ -75,37 +75,10 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const deferred = createDeferred(); this.promiseMap.set(uuid, deferred); - // create payload with testIds to send to run pytest script - const testData = JSON.stringify(testIds); - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; + traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); let runTestIdsPort: string | undefined; - const startServer = (): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - socket.on('end', () => { - traceLog('Client disconnected'); - }); - }); - - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceLog(`Server listening on port ${port}`); - resolve(port); - }); - - server.on('error', (error: Error) => { - reject(error); - }); - server.on('connection', (socket: net.Socket) => { - socket.write(payload); - traceLog('payload sent', payload); - }); - }); - - // Start the server and wait until it is listening - await startServer() + await startTestIdServer(testIds) .then((assignedPort) => { traceLog(`Server started and listening on port ${assignedPort}`); runTestIdsPort = assignedPort.toString(); From 4d1c1967afa4548847f85988873d2ef88c4ab2cc Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 13 Jun 2023 16:30:37 -0700 Subject: [PATCH 03/34] Execution adapter tests (#21421) adds test coverage for `testExecutionAdapter.ts` and `pytestExecutionAdapter.ts`. --- .../testing/testController/common/utils.ts | 64 ++-- .../pytest/pytestExecutionAdapter.ts | 16 +- .../unittest/testExecutionAdapter.ts | 16 +- .../pytestExecutionAdapter.unit.test.ts | 289 +++++++++--------- .../testExecutionAdapter.unit.test.ts | 182 ++++------- 5 files changed, 260 insertions(+), 307 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 9d426dc60ffa..232aaccd5341 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as net from 'net'; -import { traceLog } from '../../../logging'; +import { traceError, traceLog, traceVerbose } from '../../../logging'; import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; @@ -62,38 +62,52 @@ export function pythonTestAdapterRewriteEnabled(serviceContainer: IServiceContai return experiment.inExperimentSync(EnableTestAdapterRewrite.experiment); } -export const startTestIdServer = (testIds: string[]): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - // Convert the test_ids array to JSON - const testData = JSON.stringify(testIds); +export async function startTestIdServer(testIds: string[]): Promise { + const startServer = (): Promise => + new Promise((resolve, reject) => { + const server = net.createServer((socket: net.Socket) => { + // Convert the test_ids array to JSON + const testData = JSON.stringify(testIds); - // Create the headers - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; + // Create the headers + const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - // Create the payload by concatenating the headers and the test data - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; + // Create the payload by concatenating the headers and the test data + const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; - // Send the payload to the socket - socket.write(payload); + // Send the payload to the socket + socket.write(payload); - // Handle socket events - socket.on('data', (data) => { - traceLog('Received data:', data.toString()); + // Handle socket events + socket.on('data', (data) => { + traceLog('Received data:', data.toString()); + }); + + socket.on('end', () => { + traceLog('Client disconnected'); + }); }); - socket.on('end', () => { - traceLog('Client disconnected'); + server.listen(0, () => { + const { port } = server.address() as net.AddressInfo; + traceLog(`Server listening on port ${port}`); + resolve(port); }); - }); - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceLog(`Server listening on port ${port}`); - resolve(port); + server.on('error', (error: Error) => { + reject(error); + }); }); - server.on('error', (error: Error) => { - reject(error); + // Start the server and wait until it is listening + await startServer() + .then((assignedPort) => { + traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); + return assignedPort; + // if (spawnOptions.extraVariables) spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; + }) + .catch((error) => { + traceError('Error starting server for pytest test ids server:', error); }); - }); + return 0; +} diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 9c8a42ae097f..1bf032b9b594 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -118,17 +118,9 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { } traceLog(`Running PYTEST execution for the following test ids: ${testIds}`); - let pytestRunTestIdsPort: string | undefined; - await startTestIdServer(testIds) - .then((assignedPort) => { - traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); - pytestRunTestIdsPort = assignedPort.toString(); - if (spawnOptions.extraVariables) - spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; - }) - .catch((error) => { - traceError('Error starting server for pytest test ids server:', error); - }); + const pytestRunTestIdsPort = await startTestIdServer(testIds); + if (spawnOptions.extraVariables) + spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort.toString(); if (debugBool) { const pytestPort = this.testServer.getPort().toString(); @@ -140,7 +132,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { testProvider: PYTEST_PROVIDER, pytestPort, pytestUUID, - runTestIdsPort: pytestRunTestIdsPort, + runTestIdsPort: pytestRunTestIdsPort.toString(), }; traceInfo(`Running DEBUG pytest with arguments: ${testArgs.join(' ')}\r\n`); await debugLauncher!.launchDebugger(launchOptions, () => { diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index a85490c30bc8..d43832e36a1c 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -15,7 +15,7 @@ import { TestCommandOptions, TestExecutionCommand, } from '../common/types'; -import { traceLog, traceError } from '../../../logging'; +import { traceLog } from '../../../logging'; import { startTestIdServer } from '../common/utils'; /** @@ -77,19 +77,9 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { this.promiseMap.set(uuid, deferred); traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); - let runTestIdsPort: string | undefined; - await startTestIdServer(testIds) - .then((assignedPort) => { - traceLog(`Server started and listening on port ${assignedPort}`); - runTestIdsPort = assignedPort.toString(); - // Send test command to server. - // Server fire onDataReceived event once it gets response. - }) - .catch((error) => { - traceError('Error starting server:', error); - }); + const runTestIdsPort = await startTestIdServer(testIds); - await this.testServer.sendCommand(options, runTestIdsPort, () => { + await this.testServer.sendCommand(options, runTestIdsPort.toString(), () => { // disposable.dispose(); deferred.resolve(); }); diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 5e7a32ddc9e1..f60429d38e5b 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -1,141 +1,152 @@ -// /* eslint-disable @typescript-eslint/no-explicit-any */ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. -// import * as assert from 'assert'; -// import { TestRun, Uri } from 'vscode'; -// import * as typeMoq from 'typemoq'; -// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; -// import { ITestServer } from '../../../../client/testing/testController/common/types'; -// import { -// IPythonExecutionFactory, -// IPythonExecutionService, -// SpawnOptions, -// } from '../../../../client/common/process/types'; -// import { createDeferred, Deferred } from '../../../../client/common/utils/async'; -// import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; -// import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import * as assert from 'assert'; +import { TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as sinon from 'sinon'; +import * as path from 'path'; +import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +import { ITestServer } from '../../../../client/testing/testController/common/types'; +import { + IPythonExecutionFactory, + IPythonExecutionService, + SpawnOptions, +} from '../../../../client/common/process/types'; +import { createDeferred, Deferred } from '../../../../client/common/utils/async'; +import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; +import * as util from '../../../../client/testing/testController/common/utils'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; -// suite('pytest test execution adapter', () => { -// let testServer: typeMoq.IMock; -// let configService: IConfigurationService; -// let execFactory = typeMoq.Mock.ofType(); -// let adapter: PytestTestExecutionAdapter; -// let execService: typeMoq.IMock; -// let deferred: Deferred; -// let debugLauncher: typeMoq.IMock; -// setup(() => { -// testServer = typeMoq.Mock.ofType(); -// testServer.setup((t) => t.getPort()).returns(() => 12345); -// testServer -// .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// configService = ({ -// getSettings: () => ({ -// testing: { pytestArgs: ['.'] }, -// }), -// isTestExecution: () => false, -// } as unknown) as IConfigurationService; -// execFactory = typeMoq.Mock.ofType(); -// execService = typeMoq.Mock.ofType(); -// debugLauncher = typeMoq.Mock.ofType(); -// execFactory -// .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) -// .returns(() => Promise.resolve(execService.object)); -// deferred = createDeferred(); -// execService -// .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => { -// deferred.resolve(); -// return Promise.resolve({ stdout: '{}' }); -// }); -// debugLauncher -// .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => { -// deferred.resolve(); -// return Promise.resolve(); -// }); -// execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// }); -// test('pytest execution called with correct args', async () => { -// const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid123'; -// // const data = { status: 'success' }; -// testServer -// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const outputChannel = typeMoq.Mock.ofType(); -// const testRun = typeMoq.Mock.ofType(); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); -// await adapter.runTests(uri, [], false, testRun.object, execFactory.object); +suite('pytest test execution adapter', () => { + let testServer: typeMoq.IMock; + let configService: IConfigurationService; + let execFactory = typeMoq.Mock.ofType(); + let adapter: PytestTestExecutionAdapter; + let execService: typeMoq.IMock; + let deferred: Deferred; + let debugLauncher: typeMoq.IMock; + (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; + let myTestPath: string; -// const expectedArgs = [ -// '/Users/eleanorboyd/vscode-python/pythonFiles/vscode_pytest/run_pytest_script.py', -// '--rootdir', -// '/my/test/path/', -// ]; -// const expectedExtraVariables = { -// PYTHONPATH: '/Users/eleanorboyd/vscode-python/pythonFiles', -// TEST_UUID: 'uuid123', -// TEST_PORT: '12345', -// }; -// execService.verify( -// (x) => -// x.exec( -// expectedArgs, -// typeMoq.It.is((options) => { -// assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); -// assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); -// assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); -// assert.strictEqual(typeof options.extraVariables?.RUN_TEST_IDS_PORT, 'string'); -// assert.equal(options.cwd, uri.fsPath); -// assert.equal(options.throwOnStdErr, true); -// return true; -// }), -// ), -// typeMoq.Times.once(), -// ); -// }); -// test('Debug launched correctly for pytest', async () => { -// const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid123'; -// testServer -// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const outputChannel = typeMoq.Mock.ofType(); -// const testRun = typeMoq.Mock.ofType(); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); -// await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); -// debugLauncher.verify( -// (x) => -// x.launchDebugger( -// typeMoq.It.is((launchOptions) => { -// assert.equal(launchOptions.cwd, uri.fsPath); -// assert.deepEqual(launchOptions.args, ['--rootdir', '/my/test/path/', '--capture', 'no']); -// assert.equal(launchOptions.testProvider, 'pytest'); -// assert.equal(launchOptions.pytestPort, '12345'); -// assert.equal(launchOptions.pytestUUID, 'uuid123'); -// assert.strictEqual(typeof launchOptions.runTestIdsPort, 'string'); -// return true; -// }), -// typeMoq.It.isAny(), -// ), -// typeMoq.Times.once(), -// ); -// }); -// }); + setup(() => { + testServer = typeMoq.Mock.ofType(); + testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + configService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.'] }, + }), + isTestExecution: () => false, + } as unknown) as IConfigurationService; + execFactory = typeMoq.Mock.ofType(); + execService = typeMoq.Mock.ofType(); + debugLauncher = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execService.object)); + deferred = createDeferred(); + execService + .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve({ stdout: '{}' }); + }); + debugLauncher + .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve(); + }); + sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + + execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + myTestPath = path.join('/', 'my', 'test', 'path', '/'); + }); + teardown(() => { + sinon.restore(); + }); + test('pytest execution called with correct args', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + await adapter.runTests(uri, [], false, testRun.object, execFactory.object); + + const pathToPythonFiles = path.join(EXTENSION_ROOT_DIR, 'pythonFiles'); + const pathToPythonScript = path.join(pathToPythonFiles, 'vscode_pytest', 'run_pytest_script.py'); + const expectedArgs = [pathToPythonScript, '--rootdir', myTestPath]; + const expectedExtraVariables = { + PYTHONPATH: pathToPythonFiles, + TEST_UUID: 'uuid123', + TEST_PORT: '12345', + }; + // execService.verify((x) => x.exec(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); + assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); + assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); + assert.equal(options.extraVariables?.RUN_TEST_IDS_PORT, '54321'); + assert.equal(options.cwd, uri.fsPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); + }); + test('Debug launched correctly for pytest', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); + debugLauncher.verify( + (x) => + x.launchDebugger( + typeMoq.It.is((launchOptions) => { + assert.equal(launchOptions.cwd, uri.fsPath); + assert.deepEqual(launchOptions.args, ['--rootdir', myTestPath, '--capture', 'no']); + assert.equal(launchOptions.testProvider, 'pytest'); + assert.equal(launchOptions.pytestPort, '12345'); + assert.equal(launchOptions.pytestUUID, 'uuid123'); + assert.strictEqual(launchOptions.runTestIdsPort, '54321'); + return true; + }), + typeMoq.It.isAny(), + ), + typeMoq.Times.once(), + ); + }); +}); diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index e5495629bf28..7db287a3355c 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -1,118 +1,64 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. - -// import * as assert from 'assert'; -// import * as path from 'path'; -// import * as typemoq from 'typemoq'; -// import { Uri } from 'vscode'; -// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; -// import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; -// import { ITestServer, TestCommandOptions } from '../../../../client/testing/testController/common/types'; -// import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; - -// suite('Unittest test execution adapter', () => { -// let stubConfigSettings: IConfigurationService; -// let outputChannel: typemoq.IMock; - -// setup(() => { -// stubConfigSettings = ({ -// getSettings: () => ({ -// testing: { unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, -// }), -// } as unknown) as IConfigurationService; -// outputChannel = typemoq.Mock.ofType(); -// }); - -// test('runTests should send the run command to the test server', async () => { -// let options: TestCommandOptions | undefined; - -// const stubTestServer = ({ -// sendCommand(opt: TestCommandOptions, runTestIdPort?: string): Promise { -// delete opt.outChannel; -// options = opt; -// assert(runTestIdPort !== undefined); -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => '123456789', -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); -// const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); -// adapter.runTests(uri, [], false).then(() => { -// const expectedOptions: TestCommandOptions = { -// workspaceFolder: uri, -// command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, -// cwd: uri.fsPath, -// uuid: '123456789', -// debugBool: false, -// testIds: [], -// }; -// assert.deepStrictEqual(options, expectedOptions); -// }); -// }); -// test("onDataReceivedHandler should parse the data if the cwd from the payload matches the test adapter's cwd", async () => { -// const stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => '123456789', -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); -// const data = { status: 'success' }; -// const uuid = '123456789'; - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - -// // triggers runTests flow which will run onDataReceivedHandler and the -// // promise resolves into the parsed data. -// const promise = adapter.runTests(uri, [], false); - -// adapter.onDataReceivedHandler({ uuid, data: JSON.stringify(data) }); - -// const result = await promise; - -// assert.deepStrictEqual(result, data); -// }); -// test("onDataReceivedHandler should ignore the data if the cwd from the payload does not match the test adapter's cwd", async () => { -// const correctUuid = '123456789'; -// const incorrectUuid = '987654321'; -// const stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => correctUuid, -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - -// // triggers runTests flow which will run onDataReceivedHandler and the -// // promise resolves into the parsed data. -// const promise = adapter.runTests(uri, [], false); - -// const data = { status: 'success' }; -// // will not resolve due to incorrect UUID -// adapter.onDataReceivedHandler({ uuid: incorrectUuid, data: JSON.stringify(data) }); - -// const nextData = { status: 'error' }; -// // will resolve and nextData will be returned as result -// adapter.onDataReceivedHandler({ uuid: correctUuid, data: JSON.stringify(nextData) }); - -// const result = await promise; - -// assert.deepStrictEqual(result, nextData); -// }); -// }); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as path from 'path'; +import * as typemoq from 'typemoq'; +import { Uri } from 'vscode'; +import * as sinon from 'sinon'; +import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; +import { ITestServer, TestCommandOptions } from '../../../../client/testing/testController/common/types'; +import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; +import * as util from '../../../../client/testing/testController/common/utils'; + +suite('Unittest test execution adapter', () => { + let stubConfigSettings: IConfigurationService; + let outputChannel: typemoq.IMock; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, + }), + } as unknown) as IConfigurationService; + outputChannel = typemoq.Mock.ofType(); + sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + }); + teardown(() => { + sinon.restore(); + }); + + test('runTests should send the run command to the test server', async () => { + let options: TestCommandOptions | undefined; + + const stubTestServer = ({ + sendCommand(opt: TestCommandOptions, runTestIdPort?: string): Promise { + delete opt.outChannel; + options = opt; + assert(runTestIdPort !== undefined); + return Promise.resolve(); + }, + onRunDataReceived: () => { + // no body + }, + createUUID: () => '123456789', + } as unknown) as ITestServer; + + const uri = Uri.file('/foo/bar'); + const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); + + const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); + adapter.runTests(uri, [], false).then(() => { + const expectedOptions: TestCommandOptions = { + workspaceFolder: uri, + command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, + cwd: uri.fsPath, + uuid: '123456789', + debugBool: false, + testIds: [], + }; + assert.deepStrictEqual(options, expectedOptions); + }); + }); +}); From efdf65e7cd2eb18616242c211b56190938a367fe Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH 04/34] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../testController/common/resultResolver.ts | 98 +- .../testing/testController/common/utils.ts | 70 ++ .../testController/workspaceTestAdapter.ts | 13 +- src/test/mocks/vsc/index.ts | 112 +++ .../pytestExecutionAdapter.unit.test.ts | 23 +- .../resultResolver.unit.test.ts | 368 ++++++++ .../testExecutionAdapter.unit.test.ts | 5 +- .../workspaceTestAdapter.unit.test.ts | 883 ++++++++++++------ src/test/vscode-mock.ts | 2 + 9 files changed, 1200 insertions(+), 374 deletions(-) create mode 100644 src/test/testing/testController/resultResolver.unit.test.ts diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 8f8cb859b9d5..49243390ad0f 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -1,41 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { - CancellationToken, - Position, - TestController, - TestItem, - Uri, - Range, - TestMessage, - Location, - TestRun, -} from 'vscode'; +import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode'; import * as util from 'util'; -import * as path from 'path'; -import { - DiscoveredTestItem, - DiscoveredTestNode, - DiscoveredTestPayload, - ExecutionTestPayload, - ITestResultResolver, -} from './types'; +import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; import { TestProvider } from '../../types'; import { traceError, traceLog } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; -import { - DebugTestTag, - ErrorTestItemOptions, - RunTestTag, - clearAllChildren, - createErrorTestItem, - getTestCaseNodes, -} from './testItemUtilities'; +import { clearAllChildren, createErrorTestItem, getTestCaseNodes } from './testItemUtilities'; import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { splitLines } from '../../../common/stringUtils'; -import { fixLogLines } from './utils'; +import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils'; export class PythonResultResolver implements ITestResultResolver { testController: TestController; @@ -253,69 +229,5 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } } -// had to switch the order of the original parameter since required param cannot follow optional. -function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - resultResolver: ITestResultResolver, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - resultResolver.runIdToTestItem.set(child.runID, testItem); - resultResolver.runIdToVSid.set(child.runID, child.id_); - resultResolver.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, resultResolver, token); - } - } - }); -} - -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} - -export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} +// had to switch the order of the original parameter since required param cannot follow optional. diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 232aaccd5341..e92db22747c6 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,11 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as net from 'net'; +import * as path from 'path'; +import { CancellationToken, Position, TestController, TestItem, Uri, Range } from 'vscode'; import { traceError, traceLog, traceVerbose } from '../../../logging'; import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; +import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; +import { DiscoveredTestItem, DiscoveredTestNode, ITestResultResolver } from './types'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); @@ -111,3 +115,69 @@ export async function startTestIdServer(testIds: string[]): Promise { }); return 0; } + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} + +export function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5c7ee9cfe520..f3ea0b9f6193 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import * as path from 'path'; import * as util from 'util'; import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; @@ -10,10 +9,11 @@ import { traceError } from '../../logging'; import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { TestProvider } from '../types'; -import { createErrorTestItem, ErrorTestItemOptions, getTestCaseNodes } from './common/testItemUtilities'; +import { createErrorTestItem, getTestCaseNodes } from './common/testItemUtilities'; import { ITestDiscoveryAdapter, ITestExecutionAdapter, ITestResultResolver } from './common/types'; import { IPythonExecutionFactory } from '../../common/process/types'; import { ITestDebugLauncher } from '../common/types'; +import { buildErrorNodeOptions } from './common/utils'; /** * This class exposes a test-provider-agnostic way of discovering tests. @@ -162,12 +162,3 @@ export class WorkspaceTestAdapter { return Promise.resolve(); } } - -function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} diff --git a/src/test/mocks/vsc/index.ts b/src/test/mocks/vsc/index.ts index 89f4ab1a2d07..092fc67da6c6 100644 --- a/src/test/mocks/vsc/index.ts +++ b/src/test/mocks/vsc/index.ts @@ -6,6 +6,7 @@ import { EventEmitter as NodeEventEmitter } from 'events'; import * as vscode from 'vscode'; + // export * from './range'; // export * from './position'; // export * from './selection'; @@ -443,3 +444,114 @@ export enum LogLevel { */ Error = 5, } + +export class TestMessage { + /** + * Human-readable message text to display. + */ + message: string | MarkdownString; + + /** + * Expected test output. If given with {@link TestMessage.actualOutput actualOutput }, a diff view will be shown. + */ + expectedOutput?: string; + + /** + * Actual test output. If given with {@link TestMessage.expectedOutput expectedOutput }, a diff view will be shown. + */ + actualOutput?: string; + + /** + * Associated file location. + */ + location?: vscode.Location; + + /** + * Creates a new TestMessage that will present as a diff in the editor. + * @param message Message to display to the user. + * @param expected Expected output. + * @param actual Actual output. + */ + static diff(message: string | MarkdownString, expected: string, actual: string): TestMessage { + const testMessage = new TestMessage(message); + testMessage.expectedOutput = expected; + testMessage.actualOutput = actual; + return testMessage; + } + + /** + * Creates a new TestMessage instance. + * @param message The message to show to the user. + */ + constructor(message: string | MarkdownString) { + this.message = message; + } +} + +export interface TestItemCollection extends Iterable<[string, vscode.TestItem]> { + /** + * Gets the number of items in the collection. + */ + readonly size: number; + + /** + * Replaces the items stored by the collection. + * @param items Items to store. + */ + replace(items: readonly vscode.TestItem[]): void; + + /** + * Iterate over each entry in this collection. + * + * @param callback Function to execute for each entry. + * @param thisArg The `this` context used when invoking the handler function. + */ + forEach(callback: (item: vscode.TestItem, collection: TestItemCollection) => unknown, thisArg?: unknown): void; + + /** + * Adds the test item to the children. If an item with the same ID already + * exists, it'll be replaced. + * @param item Item to add. + */ + add(item: vscode.TestItem): void; + + /** + * Removes a single test item from the collection. + * @param itemId Item ID to delete. + */ + delete(itemId: string): void; + + /** + * Efficiently gets a test item by ID, if it exists, in the children. + * @param itemId Item ID to get. + * @returns The found item or undefined if it does not exist. + */ + get(itemId: string): vscode.TestItem | undefined; +} + +/** + * Represents a location inside a resource, such as a line + * inside a text file. + */ +export class Location { + /** + * The resource identifier of this location. + */ + uri: vscode.Uri; + + /** + * The document range of this location. + */ + range: vscode.Range; + + /** + * Creates a new location object. + * + * @param uri The resource identifier. + * @param rangeOrPosition The range or position. Positions will be converted to an empty range. + */ + constructor(uri: vscode.Uri, rangeOrPosition: vscode.Range) { + this.uri = uri; + this.range = rangeOrPosition; + } +} diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index f60429d38e5b..9f4c41ba8309 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -29,6 +29,7 @@ suite('pytest test execution adapter', () => { let debugLauncher: typeMoq.IMock; (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; let myTestPath: string; + let startTestIdServerStub: sinon.SinonStub; setup(() => { testServer = typeMoq.Mock.ofType(); @@ -65,7 +66,7 @@ suite('pytest test execution adapter', () => { deferred.resolve(); return Promise.resolve(); }); - sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + startTestIdServerStub = sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); @@ -75,6 +76,26 @@ suite('pytest test execution adapter', () => { teardown(() => { sinon.restore(); }); + test('startTestIdServer called with correct testIds', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + + const testIds = ['test1id', 'test2id']; + await adapter.runTests(uri, testIds, false, testRun.object, execFactory.object); + + sinon.assert.calledWithExactly(startTestIdServerStub, testIds); + }); test('pytest execution called with correct args', async () => { const uri = Uri.file(myTestPath); const uuid = 'uuid123'; diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts new file mode 100644 index 000000000000..57b321c2e36c --- /dev/null +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -0,0 +1,368 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { TestController, Uri, TestItem, CancellationToken, TestRun, TestItemCollection, Range } from 'vscode'; +import * as typemoq from 'typemoq'; +import * as sinon from 'sinon'; +import { TestProvider } from '../../../client/testing/types'; +import { + DiscoveredTestNode, + DiscoveredTestPayload, + ExecutionTestPayload, +} from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; +import * as util from '../../../client/testing/testController/common/utils'; + +suite('Result Resolver tests', () => { + suite('Test discovery', () => { + let resultResolver: ResultResolver.PythonResultResolver; + let testController: TestController; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let defaultErrorMessage: string; + let blankTestItem: TestItem; + let cancelationToken: CancellationToken; + + setup(() => { + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + + dispose: () => { + // empty + }, + } as unknown) as TestController; + defaultErrorMessage = 'pytest test discovery error (see Output > Python)'; + blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + }); + teardown(() => { + sinon.restore(); + }); + + test('resolveDiscovery calls populate test tree correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const tests: DiscoveredTestNode = { + path: 'path', + name: 'name', + type_: 'folder', + id_: 'id', + children: [], + }; + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + tests, + }; + + // stub out functionality of populateTestTreeStub which is called in resolveDiscovery + const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); + + // call resolve discovery + resultResolver.resolveDiscovery(payload, cancelationToken); + + // assert the stub functions were called with the correct parameters + + // header of populateTestTree is (testController: TestController, testTreeData: DiscoveredTestNode, testRoot: TestItem | undefined, resultResolver: ITestResultResolver, token?: CancellationToken) + sinon.assert.calledWithMatch( + populateTestTreeStub, + testController, // testController + tests, // testTreeData + undefined, // testRoot + resultResolver, // resultResolver + cancelationToken, // token + ); + }); + // what about if the error node already exists: this.testController.items.get(`DiscoveryError:${workspacePath}`); + test('resolveDiscovery should create error node on error with correct params', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const errorMessage = 'error msg A'; + const expectedErrorMessage = `${defaultErrorMessage}\r\n ${errorMessage}`; + + // stub out return values of functions called in resolveDiscovery + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + errors: [errorMessage], + }; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + + // stub out functionality of buildErrorNodeOptions and createErrorTestItem which are called in resolveDiscovery + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + + // call resolve discovery + resultResolver.resolveDiscovery(payload); + + // assert the stub functions were called with the correct parameters + + // header of buildErrorNodeOptions is (uri: Uri, message: string, testType: string) + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, workspaceUri, expectedErrorMessage, testProvider); + // header of createErrorTestItem is (options: ErrorTestItemOptions, testController: TestController, uri: Uri) + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + }); + }); + suite('Test execution result resolver', () => { + let resultResolver: ResultResolver.PythonResultResolver; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let cancelationToken: CancellationToken; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let mockTestItem1: TestItem; + let mockTestItem2: TestItem; + + setup(() => { + // create mock test items + mockTestItem1 = createMockTestItem('mockTestItem1'); + mockTestItem2 = createMockTestItem('mockTestItem2'); + + // create mock testItems to pass into a iterable + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + // create mock testItemCollection + const testItemCollectionMock = typemoq.Mock.ofType(); + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + + // create mock testController + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + + // define functions within runInstance + runInstance = typemoq.Mock.ofType(); + runInstance.setup((r) => r.name).returns(() => 'name'); + runInstance.setup((r) => r.token).returns(() => cancelationToken); + runInstance.setup((r) => r.isPersisted).returns(() => true); + runInstance + .setup((r) => r.enqueued(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('enqueue'); + return undefined; + }); + runInstance + .setup((r) => r.started(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('start'); + }); + + // mock getTestCaseNodes to just return the given testNode added + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => [testNode]); + }); + teardown(() => { + sinon.restore(); + }); + test('resolveExecution handles failed tests correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'failure', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles skipped correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'skipped', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles success correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'success', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles error correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + + const errorPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + error: 'error', + }; + + resultResolver.resolveExecution(errorPayload, runInstance.object); + + // verify that none of these functions are called + + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.never()); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = new Range(0, 0, 0, 0); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 7db287a3355c..88126225a177 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -49,14 +49,15 @@ suite('Unittest test execution adapter', () => { const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - adapter.runTests(uri, [], false).then(() => { + const testIds = ['test1id', 'test2id']; + adapter.runTests(uri, testIds, false).then(() => { const expectedOptions: TestCommandOptions = { workspaceFolder: uri, command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, cwd: uri.fsPath, uuid: '123456789', debugBool: false, - testIds: [], + testIds, }; assert.deepStrictEqual(options, expectedOptions); }); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 42e38d200546..5a2e48130746 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,267 +1,616 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. - -// import * as assert from 'assert'; -// import * as sinon from 'sinon'; -// import * as typemoq from 'typemoq'; - -// import { TestController, TestItem, Uri } from 'vscode'; -// import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; -// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 -// import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; -// import * as Telemetry from '../../../client/telemetry'; -// import { EventName } from '../../../client/telemetry/constants'; -// import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; - -// suite('Workspace test adapter', () => { -// suite('Test discovery', () => { -// let stubTestServer: ITestServer; -// let stubConfigSettings: IConfigurationService; -// let stubResultResolver: ITestResultResolver; - -// let discoverTestsStub: sinon.SinonStub; -// let sendTelemetryStub: sinon.SinonStub; -// let outputChannel: typemoq.IMock; - -// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - -// // Stubbed test controller (see comment around L.40) -// let testController: TestController; -// let log: string[] = []; - -// const sandbox = sinon.createSandbox(); - -// setup(() => { -// stubConfigSettings = ({ -// getSettings: () => ({ -// testing: { unittestArgs: ['--foo'] }, -// }), -// } as unknown) as IConfigurationService; - -// stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// } as unknown) as ITestServer; - -// stubResultResolver = ({ -// resolveDiscovery: () => { -// // no body -// }, -// resolveExecution: () => { -// // no body -// }, -// vsIdToRunId: { -// get: sinon.stub().returns('expectedRunId'), -// }, -// } as unknown) as ITestResultResolver; - -// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); -// // const expectedRunId = 'expectedRunId'; -// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - -// // For some reason the 'tests' namespace in vscode returns undefined. -// // While I figure out how to expose to the tests, they will run -// // against a stub test controller and stub test items. -// const testItem = ({ -// canResolveChildren: false, -// tags: [], -// children: { -// add: () => { -// // empty -// }, -// }, -// } as unknown) as TestItem; - -// testController = ({ -// items: { -// get: () => { -// log.push('get'); -// }, -// add: () => { -// log.push('add'); -// }, -// replace: () => { -// log.push('replace'); -// }, -// delete: () => { -// log.push('delete'); -// }, -// }, -// createTestItem: () => { -// log.push('createTestItem'); -// return testItem; -// }, -// dispose: () => { -// // empty -// }, -// } as unknown) as TestController; - -// // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - -// const mockSendTelemetryEvent = ( -// eventName: EventName, -// _: number | Record | undefined, -// properties: unknown, -// ) => { -// telemetryEvent.push({ -// eventName, -// properties: properties as Record, -// }); -// }; - -// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); -// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); -// outputChannel = typemoq.Mock.ofType(); -// }); - -// teardown(() => { -// telemetryEvent = []; -// log = []; -// testController.dispose(); -// sandbox.restore(); -// }); - -// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { -// discoverTestsStub.resolves(); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { -// discoverTestsStub.callsFake( -// async () => -// new Promise((resolve) => { -// setTimeout(() => { -// // Simulate time taken by discovery. -// resolve(); -// }, 2000); -// }), -// ); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// // Try running discovery twice -// const one = workspaceTestAdapter.discoverTests(testController); -// const two = workspaceTestAdapter.discoverTests(testController); - -// Promise.all([one, two]); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { -// discoverTestsStub.resolves({ status: 'success' }); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.strictEqual(lastEvent.properties.failed, false); -// }); - -// test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { -// discoverTestsStub.rejects(new Error('foo')); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.ok(lastEvent.properties.failed); - -// assert.deepStrictEqual(log, ['createTestItem', 'add']); -// }); - -// /** -// * TODO To test: -// * - successful discovery but no data: delete everything from the test controller -// * - successful discovery with error status: add error node to tree -// * - single root: populate tree if there's no root node -// * - single root: update tree if there's a root node -// * - single root: delete tree if there are no tests in the test data -// * - multiroot: update the correct folders -// */ -// }); -// }); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; + +import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 +import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +import * as Telemetry from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as util from '../../../client/testing/testController/common/utils'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; + +suite('Workspace test adapter', () => { + suite('Test discovery', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + } as unknown) as ITestResultResolver; + + // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); + // const expectedRunId = 'expectedRunId'; + // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); + sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sinon.restore(); + }); + + test('If discovery failed correctly create error node', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + const abc = await workspaceTestAdapter.discoverTests(testController); + console.log(abc); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { + discoverTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); + + Promise.all([one, two]); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.strictEqual(lastEvent.properties.failed, false); + }); + + test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.ok(lastEvent.properties.failed); + }); + }); + suite('Test execution workspace test adapter', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + let executionTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + let resultResolver: ResultResolver.PythonResultResolver; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + const sandbox = sinon.createSandbox(); + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + vsIdToRunId: { + get: sinon.stub().returns('expectedRunId'), + }, + } as unknown) as ITestResultResolver; + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + executionTestsStub = sandbox.stub(UnittestTestExecutionAdapter.prototype, 'runTests'); + sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + runInstance = typemoq.Mock.ofType(); + + const testProvider = 'pytest'; + const workspaceUri = Uri.file('foo'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sandbox.restore(); + }); + test('When executing tests, the right tests should be sent to be executed', async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + resultResolver, + ); + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => + // Custom implementation logic here based on the provided testNode and collection + + // Example implementation: returning a predefined array of TestItem objects + [testNode], + ); + + const mockTestItem1 = createMockTestItem('mockTestItem1'); + const mockTestItem2 = createMockTestItem('mockTestItem2'); + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + // Add as many mock TestItems as needed + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + const testItemCollectionMock = typemoq.Mock.ofType(); + + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, [mockTestItem1, mockTestItem2]); + + runInstance.verify((r) => r.started(typemoq.It.isAny()), typemoq.Times.exactly(2)); + }); + + test("When executing tests, the workspace test adapter should call the test execute adapter's executionTest method", async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution is already running, do not call executionAdapter.runTests again', async () => { + executionTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + Promise.all([one, two]); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution failed correctly create error node', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_RUN_ALL_FAILED); + assert.strictEqual(telemetryEvent.length, 1); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = typemoq.Mock.ofType(); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/vscode-mock.ts b/src/test/vscode-mock.ts index ebbe7ca59e72..44518e7575a7 100644 --- a/src/test/vscode-mock.ts +++ b/src/test/vscode-mock.ts @@ -70,6 +70,8 @@ mockedVSCode.Hover = vscodeMocks.Hover; mockedVSCode.Disposable = vscodeMocks.Disposable as any; mockedVSCode.ExtensionKind = vscodeMocks.ExtensionKind; mockedVSCode.CodeAction = vscodeMocks.CodeAction; +mockedVSCode.TestMessage = vscodeMocks.TestMessage; +mockedVSCode.Location = vscodeMocks.Location; mockedVSCode.EventEmitter = vscodeMocks.EventEmitter; mockedVSCode.CancellationTokenSource = vscodeMocks.CancellationTokenSource; mockedVSCode.CompletionItemKind = vscodeMocks.CompletionItemKind; From 0bde912e606b5dc97e783394d6bf4d2fcf24f21f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 20 Jun 2023 11:22:33 -0700 Subject: [PATCH 05/34] fix issue regarding port return and await --- .../testing/testController/common/utils.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index e92db22747c6..f98550d3e72b 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -104,16 +104,23 @@ export async function startTestIdServer(testIds: string[]): Promise { }); // Start the server and wait until it is listening - await startServer() - .then((assignedPort) => { - traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); - return assignedPort; - // if (spawnOptions.extraVariables) spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; - }) - .catch((error) => { - traceError('Error starting server for pytest test ids server:', error); - }); - return 0; + let returnPort = 0; + try { + await startServer() + .then((assignedPort) => { + traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); + returnPort = assignedPort; + }) + .catch((error) => { + traceError('Error starting server for pytest test ids server:', error); + return 0; + }) + .finally(() => returnPort); + return returnPort; + } catch { + traceError('Error starting server for pytest test ids server, cannot get port.'); + return returnPort; + } } export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { From 776ae23b5c22777f7eec95cf532212c39920d462 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 20 Jun 2023 14:08:38 -0700 Subject: [PATCH 06/34] remove unneeded prints --- pythonFiles/unittestadapter/execution.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 4695064396cc..17c125e5843a 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -242,7 +242,6 @@ def run_tests( try: client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client_socket.connect(("localhost", run_test_ids_port_int)) - print(f"CLIENT: Server listening on port {run_test_ids_port_int}...") buffer = b"" while True: @@ -263,7 +262,6 @@ def run_tests( buffer = b"" # Process the JSON data - print(f"Received JSON data: {test_ids_from_buffer}") break except json.JSONDecodeError: # JSON decoding error, the complete JSON object is not yet received From f4da1c4226dee8efcd1d7033ea17cba5f70a87f9 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 11:15:23 -0700 Subject: [PATCH 07/34] Remote resolver (#21332) This is branch will serve as a feature branch for all changes related to switching to the remote resolver. This will include - switching from using the testAdapter to parse the return data to now using this new class resultResolver - adding tests for all testAdapters, fixing for server and adding for resultResolver - moving sendCommand to a new file, out of the server, and getting pytest to adopt it - moving the server which send the test IDs to a new file and adopt it for both pytest and unittest - write tests for these two new files. --- src/client/testing/common/socketServer.ts | 2 +- .../testController/common/resultResolver.ts | 321 +++++++++++ .../testing/testController/common/server.ts | 43 +- .../testing/testController/common/types.ts | 12 +- .../testing/testController/controller.ts | 16 +- .../pytest/pytestDiscoveryAdapter.ts | 73 +-- .../pytest/pytestExecutionAdapter.ts | 49 +- .../unittest/testDiscoveryAdapter.ts | 36 +- .../unittest/testExecutionAdapter.ts | 47 +- .../testController/workspaceTestAdapter.ts | 295 +--------- .../pytestDiscoveryAdapter.unit.test.ts | 123 +++-- .../pytestExecutionAdapter.unit.test.ts | 123 +++-- .../testController/server.unit.test.ts | 507 +++++++++++------ .../testDiscoveryAdapter.unit.test.ts | 62 +-- .../workspaceTestAdapter.unit.test.ts | 513 +++++++++--------- 15 files changed, 1301 insertions(+), 921 deletions(-) create mode 100644 src/client/testing/testController/common/resultResolver.ts diff --git a/src/client/testing/common/socketServer.ts b/src/client/testing/common/socketServer.ts index 554d8c8a0c76..c27bf5a1606c 100644 --- a/src/client/testing/common/socketServer.ts +++ b/src/client/testing/common/socketServer.ts @@ -123,7 +123,7 @@ export class UnitTestSocketServer extends EventEmitter implements IUnitTestSocke if ((socket as any).id) { destroyedSocketId = (socket as any).id; } - this.log('socket disconnected', destroyedSocketId.toString()); + this.log('socket disconnected', destroyedSocketId?.toString()); if (socket && socket.destroy) { socket.destroy(); } diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts new file mode 100644 index 000000000000..8f8cb859b9d5 --- /dev/null +++ b/src/client/testing/testController/common/resultResolver.ts @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { + CancellationToken, + Position, + TestController, + TestItem, + Uri, + Range, + TestMessage, + Location, + TestRun, +} from 'vscode'; +import * as util from 'util'; +import * as path from 'path'; +import { + DiscoveredTestItem, + DiscoveredTestNode, + DiscoveredTestPayload, + ExecutionTestPayload, + ITestResultResolver, +} from './types'; +import { TestProvider } from '../../types'; +import { traceError, traceLog } from '../../../logging'; +import { Testing } from '../../../common/utils/localize'; +import { + DebugTestTag, + ErrorTestItemOptions, + RunTestTag, + clearAllChildren, + createErrorTestItem, + getTestCaseNodes, +} from './testItemUtilities'; +import { sendTelemetryEvent } from '../../../telemetry'; +import { EventName } from '../../../telemetry/constants'; +import { splitLines } from '../../../common/stringUtils'; +import { fixLogLines } from './utils'; + +export class PythonResultResolver implements ITestResultResolver { + testController: TestController; + + testProvider: TestProvider; + + public runIdToTestItem: Map; + + public runIdToVSid: Map; + + public vsIdToRunId: Map; + + constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) { + this.testController = testController; + this.testProvider = testProvider; + + this.runIdToTestItem = new Map(); + this.runIdToVSid = new Map(); + this.vsIdToRunId = new Map(); + } + + public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { + const workspacePath = this.workspaceUri.fsPath; + traceLog('Using result resolver for discovery'); + + const rawTestData = payload; + if (!rawTestData) { + // No test data is available + return Promise.resolve(); + } + + // Check if there were any errors in the discovery process. + if (rawTestData.status === 'error') { + const testingErrorConst = + this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery; + const { errors } = rawTestData; + traceError(testingErrorConst, '\r\n', errors!.join('\r\n\r\n')); + + let errorNode = this.testController.items.get(`DiscoveryError:${workspacePath}`); + const message = util.format( + `${testingErrorConst} ${Testing.seePythonOutput}\r\n`, + errors!.join('\r\n\r\n'), + ); + + if (errorNode === undefined) { + const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider); + errorNode = createErrorTestItem(this.testController, options); + this.testController.items.add(errorNode); + } + errorNode.error = message; + } else { + // Remove the error node if necessary, + // then parse and insert test data. + this.testController.items.delete(`DiscoveryError:${workspacePath}`); + + if (rawTestData.tests) { + // If the test root for this folder exists: Workspace refresh, update its children. + // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree. + populateTestTree(this.testController, rawTestData.tests, undefined, this, token); + } else { + // Delete everything from the test controller. + this.testController.items.replace([]); + } + } + + sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { + tool: this.testProvider, + failed: false, + }); + return Promise.resolve(); + } + + public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { + const rawTestExecData = payload; + if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { + // Map which holds the subtest information for each test item. + const subTestStats: Map = new Map(); + + // iterate through payload and update the UI accordingly. + for (const keyTemp of Object.keys(rawTestExecData.result)) { + const testCases: TestItem[] = []; + + // grab leaf level test items + this.testController.items.forEach((i) => { + const tempArr: TestItem[] = getTestCaseNodes(i); + testCases.push(...tempArr); + }); + + if ( + rawTestExecData.result[keyTemp].outcome === 'failure' || + rawTestExecData.result[keyTemp].outcome === 'passed-unexpected' + ) { + const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? ''; + const traceback = splitLines(rawTraceback, { + trim: false, + removeEmptyEntries: true, + }).join('\r\n'); + + const text = `${rawTestExecData.result[keyTemp].test} failed: ${ + rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome + }\r\n${traceback}\r\n`; + const message = new TestMessage(text); + + // note that keyTemp is a runId for unittest library... + const grabVSid = this.runIdToVSid.get(keyTemp); + // search through freshly built array of testItem to find the failed test and update UI. + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + message.location = new Location(indiItem.uri, indiItem.range); + runInstance.failed(indiItem, message); + runInstance.appendOutput(fixLogLines(text)); + } + } + }); + } else if ( + rawTestExecData.result[keyTemp].outcome === 'success' || + rawTestExecData.result[keyTemp].outcome === 'expected-failure' + ) { + const grabTestItem = this.runIdToTestItem.get(keyTemp); + const grabVSid = this.runIdToVSid.get(keyTemp); + if (grabTestItem !== undefined) { + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + runInstance.passed(grabTestItem); + runInstance.appendOutput('Passed here'); + } + } + }); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'skipped') { + const grabTestItem = this.runIdToTestItem.get(keyTemp); + const grabVSid = this.runIdToVSid.get(keyTemp); + if (grabTestItem !== undefined) { + testCases.forEach((indiItem) => { + if (indiItem.id === grabVSid) { + if (indiItem.uri && indiItem.range) { + runInstance.skipped(grabTestItem); + runInstance.appendOutput('Skipped here'); + } + } + }); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') { + // split on " " since the subtest ID has the parent test ID in the first part of the ID. + const parentTestCaseId = keyTemp.split(' ')[0]; + const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); + const data = rawTestExecData.result[keyTemp]; + // find the subtest's parent test item + if (parentTestItem) { + const subtestStats = subTestStats.get(parentTestCaseId); + if (subtestStats) { + subtestStats.failed += 1; + } else { + subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); + runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); + // clear since subtest items don't persist between runs + clearAllChildren(parentTestItem); + } + const subtestId = keyTemp; + const subTestItem = this.testController?.createTestItem(subtestId, subtestId); + runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`)); + // create a new test item for the subtest + if (subTestItem) { + const traceback = data.traceback ?? ''; + const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`; + runInstance.appendOutput(fixLogLines(text)); + parentTestItem.children.add(subTestItem); + runInstance.started(subTestItem); + const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? ''); + if (parentTestItem.uri && parentTestItem.range) { + message.location = new Location(parentTestItem.uri, parentTestItem.range); + } + runInstance.failed(subTestItem, message); + } else { + throw new Error('Unable to create new child node for subtest'); + } + } else { + throw new Error('Parent test item not found'); + } + } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') { + // split on " " since the subtest ID has the parent test ID in the first part of the ID. + const parentTestCaseId = keyTemp.split(' ')[0]; + const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); + + // find the subtest's parent test item + if (parentTestItem) { + const subtestStats = subTestStats.get(parentTestCaseId); + if (subtestStats) { + subtestStats.passed += 1; + } else { + subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); + runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); + // clear since subtest items don't persist between runs + clearAllChildren(parentTestItem); + } + const subtestId = keyTemp; + const subTestItem = this.testController?.createTestItem(subtestId, subtestId); + // create a new test item for the subtest + if (subTestItem) { + parentTestItem.children.add(subTestItem); + runInstance.started(subTestItem); + runInstance.passed(subTestItem); + runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`)); + } else { + throw new Error('Unable to create new child node for subtest'); + } + } else { + throw new Error('Parent test item not found'); + } + } + } + } + return Promise.resolve(); + } +} +// had to switch the order of the original parameter since required param cannot follow optional. +function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 6bd9bf348e20..32829e355ccb 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -9,7 +9,7 @@ import { IPythonExecutionFactory, SpawnOptions, } from '../../../common/process/types'; -import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; +import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; @@ -24,6 +24,10 @@ export class PythonTestServer implements ITestServer, Disposable { private ready: Promise; + private _onRunDataReceived: EventEmitter = new EventEmitter(); + + private _onDiscoveryDataReceived: EventEmitter = new EventEmitter(); + constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) { this.server = net.createServer((socket: net.Socket) => { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data @@ -48,11 +52,28 @@ export class PythonTestServer implements ITestServer, Disposable { rawData = rpcHeaders.remainingRawData; const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData); const extractedData = rpcContent.extractedJSON; + // do not send until we have the full content if (extractedData.length === Number(totalContentLength)) { - // do not send until we have the full content - traceVerbose(`Received data from test server: ${extractedData}`); - this._onDataReceived.fire({ uuid, data: extractedData }); - this.uuids = this.uuids.filter((u) => u !== uuid); + // if the rawData includes tests then this is a discovery request + if (rawData.includes(`"tests":`)) { + this._onDiscoveryDataReceived.fire({ + uuid, + data: rpcContent.extractedJSON, + }); + // if the rawData includes result then this is a run request + } else if (rawData.includes(`"result":`)) { + this._onRunDataReceived.fire({ + uuid, + data: rpcContent.extractedJSON, + }); + } else { + traceLog( + `Error processing test server request: request is not recognized as discovery or run.`, + ); + this._onDataReceived.fire({ uuid: '', data: '' }); + return; + } + // this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO?? buffer = Buffer.alloc(0); } else { break; @@ -97,6 +118,18 @@ export class PythonTestServer implements ITestServer, Disposable { return uuid; } + public deleteUUID(uuid: string): void { + this.uuids = this.uuids.filter((u) => u !== uuid); + } + + public get onRunDataReceived(): Event { + return this._onRunDataReceived.event; + } + + public get onDiscoveryDataReceived(): Event { + return this._onDiscoveryDataReceived.event; + } + public dispose(): void { this.server.close(); this._onDataReceived.dispose(); diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 4307d7a3913f..cb7fda797c4a 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -172,12 +172,21 @@ export type TestCommandOptionsPytest = { */ export interface ITestServer { readonly onDataReceived: Event; + readonly onRunDataReceived: Event; + readonly onDiscoveryDataReceived: Event; sendCommand(options: TestCommandOptions, runTestIdsPort?: string, callback?: () => void): Promise; serverReady(): Promise; getPort(): number; createUUID(cwd: string): string; + deleteUUID(uuid: string): void; +} +export interface ITestResultResolver { + runIdToVSid: Map; + runIdToTestItem: Map; + vsIdToRunId: Map; + resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; + resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; } - export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature discoverTests(uri: Uri): Promise; @@ -192,6 +201,7 @@ export interface ITestExecutionAdapter { uri: Uri, testIds: string[], debugBool?: boolean, + runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise; diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 0d3487855380..eff333a4cdd9 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -31,12 +31,14 @@ import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../common/constants'; import { TestProvider } from '../types'; import { PythonTestServer } from './common/server'; import { DebugTestTag, getNodeByUri, RunTestTag } from './common/testItemUtilities'; +import { pythonTestAdapterRewriteEnabled } from './common/utils'; import { ITestController, ITestDiscoveryAdapter, ITestFrameworkController, TestRefreshOptions, ITestExecutionAdapter, + ITestResultResolver, } from './common/types'; import { UnittestTestDiscoveryAdapter } from './unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from './unittest/testExecutionAdapter'; @@ -44,8 +46,8 @@ import { PytestTestDiscoveryAdapter } from './pytest/pytestDiscoveryAdapter'; import { PytestTestExecutionAdapter } from './pytest/pytestExecutionAdapter'; import { WorkspaceTestAdapter } from './workspaceTestAdapter'; import { ITestDebugLauncher } from '../common/types'; -import { pythonTestAdapterRewriteEnabled } from './common/utils'; import { IServiceContainer } from '../../ioc/types'; +import { PythonResultResolver } from './common/resultResolver'; // Types gymnastics to make sure that sendTriggerTelemetry only accepts the correct types. type EventPropertyType = IEventNamePropertyMapping[EventName.UNITTEST_DISCOVERY_TRIGGER]; @@ -161,30 +163,37 @@ export class PythonTestController implements ITestController, IExtensionSingleAc let discoveryAdapter: ITestDiscoveryAdapter; let executionAdapter: ITestExecutionAdapter; let testProvider: TestProvider; + let resultResolver: ITestResultResolver; if (settings.testing.unittestEnabled) { + testProvider = UNITTEST_PROVIDER; + resultResolver = new PythonResultResolver(this.testController, testProvider, workspace.uri); discoveryAdapter = new UnittestTestDiscoveryAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); executionAdapter = new UnittestTestExecutionAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); - testProvider = UNITTEST_PROVIDER; } else { + testProvider = PYTEST_PROVIDER; + resultResolver = new PythonResultResolver(this.testController, testProvider, workspace.uri); discoveryAdapter = new PytestTestDiscoveryAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); executionAdapter = new PytestTestExecutionAdapter( this.pythonTestServer, this.configSettings, this.testOutputChannel, + resultResolver, ); - testProvider = PYTEST_PROVIDER; } const workspaceTestAdapter = new WorkspaceTestAdapter( @@ -192,6 +201,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc discoveryAdapter, executionAdapter, workspace.uri, + resultResolver, ); this.testAdapters.set(workspace.uri, workspaceTestAdapter); diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index aeb920407cd2..4378c68b534c 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -10,8 +10,14 @@ import { import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; -import { traceError, traceLog, traceVerbose } from '../../../logging'; -import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, ITestServer } from '../common/types'; +import { traceError, traceVerbose } from '../../../logging'; +import { + DataReceivedEvent, + DiscoveredTestPayload, + ITestDiscoveryAdapter, + ITestResultResolver, + ITestServer, +} from '../common/types'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. #this seems incorrectly copied @@ -19,39 +25,32 @@ import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, ITestS export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { private promiseMap: Map> = new Map(); - private deferred: Deferred | undefined; - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} - discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { - if (executionFactory !== undefined) { - // ** new version of discover tests. - const settings = this.configSettings.getSettings(uri); - const { pytestArgs } = settings.testing; - traceVerbose(pytestArgs); - return this.runPytestDiscovery(uri, executionFactory); + async discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { + const settings = this.configSettings.getSettings(uri); + const { pytestArgs } = settings.testing; + traceVerbose(pytestArgs); + const disposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { + // cancelation token ? + this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + }); + try { + await this.runPytestDiscovery(uri, executionFactory); + } finally { + disposable.dispose(); } - // if executionFactory is undefined, we are using the old method signature of discover tests. - traceVerbose(uri); - this.deferred = createDeferred(); - return this.deferred.promise; + // this is only a placeholder to handle function overloading until rewrite is finished + const discoveryPayload: DiscoveredTestPayload = { cwd: uri.fsPath, status: 'success' }; + return discoveryPayload; } - async runPytestDiscovery(uri: Uri, executionFactory: IPythonExecutionFactory): Promise { + async runPytestDiscovery(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { const deferred = createDeferred(); const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); @@ -79,13 +78,19 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { allowEnvironmentFetchExceptions: false, resource: uri, }; - const execService = await executionFactory.createActivatedEnvironment(creationOptions); - const discoveryArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs); - traceLog(`Discovering pytest tests with arguments: ${discoveryArgs.join(' ')}`); - execService.exec(discoveryArgs, spawnOptions).catch((ex) => { - traceError(`Error occurred while discovering tests: ${ex}`); - deferred.reject(ex as Error); - }); + const execService = await executionFactory?.createActivatedEnvironment(creationOptions); + // delete UUID following entire discovery finishing. + execService + ?.exec(['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs), spawnOptions) + .then(() => { + this.testServer.deleteUUID(uuid); + return deferred.resolve(); + }) + .catch((err) => { + traceError(`Error while trying to run pytest discovery, \n${err}\r\n\r\n`); + this.testServer.deleteUUID(uuid); + return deferred.reject(err); + }); return deferred.promise; } } diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 90704b5d67f4..6d63b49d007f 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -1,13 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { Uri } from 'vscode'; +import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; -import { DataReceivedEvent, ExecutionTestPayload, ITestExecutionAdapter, ITestServer } from '../common/types'; +import { + DataReceivedEvent, + ExecutionTestPayload, + ITestExecutionAdapter, + ITestResultResolver, + ITestServer, +} from '../common/types'; import { ExecutionFactoryCreateWithEnvironmentOptions, IPythonExecutionFactory, @@ -27,39 +33,35 @@ import { EXTENSION_ROOT_DIR } from '../../../common/constants'; export class PytestTestExecutionAdapter implements ITestExecutionAdapter { private promiseMap: Map> = new Map(); - private deferred: Deferred | undefined; - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} async runTests( uri: Uri, testIds: string[], debugBool?: boolean, + runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { - if (executionFactory !== undefined) { - // ** new version of run tests. - return this.runTestsNew(uri, testIds, debugBool, executionFactory, debugLauncher); + traceVerbose(uri, testIds, debugBool); + const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { + if (runInstance) { + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + } + }); + try { + await this.runTestsNew(uri, testIds, debugBool, executionFactory, debugLauncher); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) } - // if executionFactory is undefined, we are using the old method signature of run tests. - this.outputChannel.appendLine('Running tests.'); - this.deferred = createDeferred(); - return this.deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } private async runTestsNew( @@ -187,6 +189,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { return Promise.reject(ex); } - return deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } } diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 9c565af78c08..8d393a8da18d 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -10,11 +10,11 @@ import { DataReceivedEvent, DiscoveredTestPayload, ITestDiscoveryAdapter, + ITestResultResolver, ITestServer, TestCommandOptions, TestDiscoveryCommand, } from '../common/types'; -import { traceInfo } from '../../../logging'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. @@ -28,17 +28,8 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); - } - } + private readonly resultResolver?: ITestResultResolver, + ) {} public async discoverTests(uri: Uri): Promise { const deferred = createDeferred(); @@ -60,12 +51,23 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { this.promiseMap.set(uuid, deferred); - // Send the test command to the server. - // The server will fire an onDataReceived event once it gets a response. - traceInfo(`Sending discover unittest script to server.`); - this.testServer.sendCommand(options); + const disposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { + this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + }); + try { + await this.callSendCommand(options); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) + } + const discoveryPayload: DiscoveredTestPayload = { cwd: uri.fsPath, status: 'success' }; + return discoveryPayload; + } - return deferred.promise; + private async callSendCommand(options: TestCommandOptions): Promise { + await this.testServer.sendCommand(options); + const discoveryPayload: DiscoveredTestPayload = { cwd: '', status: 'success' }; + return discoveryPayload; } } diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index b671a64138cb..b18b5945c6b0 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as path from 'path'; -import { Uri } from 'vscode'; +import { TestRun, Uri } from 'vscode'; import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; @@ -11,6 +11,7 @@ import { DataReceivedEvent, ExecutionTestPayload, ITestExecutionAdapter, + ITestResultResolver, ITestServer, TestCommandOptions, TestExecutionCommand, @@ -30,19 +31,31 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { public testServer: ITestServer, public configSettings: IConfigurationService, private readonly outputChannel: ITestOutputChannel, - ) { - testServer.onDataReceived(this.onDataReceivedHandler, this); - } - - public onDataReceivedHandler({ uuid, data }: DataReceivedEvent): void { - const deferred = this.promiseMap.get(uuid); - if (deferred) { - deferred.resolve(JSON.parse(data)); - this.promiseMap.delete(uuid); + private readonly resultResolver?: ITestResultResolver, + ) {} + + public async runTests( + uri: Uri, + testIds: string[], + debugBool?: boolean, + runInstance?: TestRun, + ): Promise { + const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { + if (runInstance) { + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + } + }); + try { + await this.runTestsNew(uri, testIds, debugBool); + } finally { + disposable.dispose(); + // confirm with testing that this gets called (it must clean this up) } + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } - public async runTests(uri: Uri, testIds: string[], debugBool?: boolean): Promise { + private async runTestsNew(uri: Uri, testIds: string[], debugBool?: boolean): Promise { const settings = this.configSettings.getSettings(uri); const { cwd, unittestArgs } = settings.testing; @@ -62,7 +75,6 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const deferred = createDeferred(); this.promiseMap.set(uuid, deferred); - // create payload with testIds to send to run pytest script const testData = JSON.stringify(testIds); const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; @@ -99,15 +111,18 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { runTestIdsPort = assignedPort.toString(); // Send test command to server. // Server fire onDataReceived event once it gets response. - this.testServer.sendCommand(options, runTestIdsPort, () => { - deferred.resolve(); - }); }) .catch((error) => { traceError('Error starting server:', error); }); - return deferred.promise; + await this.testServer.sendCommand(options, runTestIdsPort, () => { + // disposable.dispose(); + deferred.resolve(); + }); + // return deferred.promise; + const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + return executionPayload; } } diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5cba6c193d3c..5c7ee9cfe520 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -3,34 +3,15 @@ import * as path from 'path'; import * as util from 'util'; -import { - CancellationToken, - Position, - Range, - TestController, - TestItem, - TestMessage, - TestRun, - Uri, - Location, -} from 'vscode'; -import { splitLines } from '../../common/stringUtils'; +import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; import { Testing } from '../../common/utils/localize'; -import { traceError, traceVerbose } from '../../logging'; +import { traceError } from '../../logging'; import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { TestProvider } from '../types'; -import { - clearAllChildren, - createErrorTestItem, - DebugTestTag, - ErrorTestItemOptions, - getTestCaseNodes, - RunTestTag, -} from './common/testItemUtilities'; -import { DiscoveredTestItem, DiscoveredTestNode, ITestDiscoveryAdapter, ITestExecutionAdapter } from './common/types'; -import { fixLogLines } from './common/utils'; +import { createErrorTestItem, ErrorTestItemOptions, getTestCaseNodes } from './common/testItemUtilities'; +import { ITestDiscoveryAdapter, ITestExecutionAdapter, ITestResultResolver } from './common/types'; import { IPythonExecutionFactory } from '../../common/process/types'; import { ITestDebugLauncher } from '../common/types'; @@ -48,22 +29,13 @@ export class WorkspaceTestAdapter { private executing: Deferred | undefined; - runIdToTestItem: Map; - - runIdToVSid: Map; - - vsIdToRunId: Map; - constructor( private testProvider: TestProvider, private discoveryAdapter: ITestDiscoveryAdapter, private executionAdapter: ITestExecutionAdapter, private workspaceUri: Uri, - ) { - this.runIdToTestItem = new Map(); - this.runIdToVSid = new Map(); - this.vsIdToRunId = new Map(); - } + private resultResolver: ITestResultResolver, + ) {} public async executeTests( testController: TestController, @@ -81,7 +53,6 @@ export class WorkspaceTestAdapter { const deferred = createDeferred(); this.executing = deferred; - let rawTestExecData; const testCaseNodes: TestItem[] = []; const testCaseIdsSet = new Set(); try { @@ -93,7 +64,7 @@ export class WorkspaceTestAdapter { // iterate through testItems nodes and fetch their unittest runID to pass in as argument testCaseNodes.forEach((node) => { runInstance.started(node); // do the vscode ui test item start here before runtest - const runId = this.vsIdToRunId.get(node.id); + const runId = this.resultResolver.vsIdToRunId.get(node.id); if (runId) { testCaseIdsSet.add(runId); } @@ -101,16 +72,16 @@ export class WorkspaceTestAdapter { const testCaseIds = Array.from(testCaseIdsSet); // ** execution factory only defined for new rewrite way if (executionFactory !== undefined) { - traceVerbose('executionFactory defined'); - rawTestExecData = await this.executionAdapter.runTests( + await this.executionAdapter.runTests( this.workspaceUri, testCaseIds, debugBool, + runInstance, executionFactory, debugLauncher, ); } else { - rawTestExecData = await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, debugBool); + await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, debugBool); } deferred.resolve(); } catch (ex) { @@ -136,146 +107,6 @@ export class WorkspaceTestAdapter { this.executing = undefined; } - if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { - // Map which holds the subtest information for each test item. - const subTestStats: Map = new Map(); - - // iterate through payload and update the UI accordingly. - for (const keyTemp of Object.keys(rawTestExecData.result)) { - const testCases: TestItem[] = []; - - // grab leaf level test items - testController.items.forEach((i) => { - const tempArr: TestItem[] = getTestCaseNodes(i); - testCases.push(...tempArr); - }); - - if ( - rawTestExecData.result[keyTemp].outcome === 'failure' || - rawTestExecData.result[keyTemp].outcome === 'passed-unexpected' - ) { - const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? ''; - const traceback = splitLines(rawTraceback, { - trim: false, - removeEmptyEntries: true, - }).join('\r\n'); - - const text = `${rawTestExecData.result[keyTemp].test} failed: ${ - rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome - }\r\n${traceback}\r\n`; - const message = new TestMessage(text); - - // note that keyTemp is a runId for unittest library... - const grabVSid = this.runIdToVSid.get(keyTemp); - // search through freshly built array of testItem to find the failed test and update UI. - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - message.location = new Location(indiItem.uri, indiItem.range); - runInstance.failed(indiItem, message); - runInstance.appendOutput(fixLogLines(text)); - } - } - }); - } else if ( - rawTestExecData.result[keyTemp].outcome === 'success' || - rawTestExecData.result[keyTemp].outcome === 'expected-failure' - ) { - const grabTestItem = this.runIdToTestItem.get(keyTemp); - const grabVSid = this.runIdToVSid.get(keyTemp); - if (grabTestItem !== undefined) { - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - runInstance.passed(grabTestItem); - runInstance.appendOutput('Passed here'); - } - } - }); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'skipped') { - const grabTestItem = this.runIdToTestItem.get(keyTemp); - const grabVSid = this.runIdToVSid.get(keyTemp); - if (grabTestItem !== undefined) { - testCases.forEach((indiItem) => { - if (indiItem.id === grabVSid) { - if (indiItem.uri && indiItem.range) { - runInstance.skipped(grabTestItem); - runInstance.appendOutput('Skipped here'); - } - } - }); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') { - // split on " " since the subtest ID has the parent test ID in the first part of the ID. - const parentTestCaseId = keyTemp.split(' ')[0]; - const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); - const data = rawTestExecData.result[keyTemp]; - // find the subtest's parent test item - if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); - if (subtestStats) { - subtestStats.failed += 1; - } else { - subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); - runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); - // clear since subtest items don't persist between runs - clearAllChildren(parentTestItem); - } - const subtestId = keyTemp; - const subTestItem = testController?.createTestItem(subtestId, subtestId); - runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`)); - // create a new test item for the subtest - if (subTestItem) { - const traceback = data.traceback ?? ''; - const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`; - runInstance.appendOutput(fixLogLines(text)); - parentTestItem.children.add(subTestItem); - runInstance.started(subTestItem); - const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? ''); - if (parentTestItem.uri && parentTestItem.range) { - message.location = new Location(parentTestItem.uri, parentTestItem.range); - } - runInstance.failed(subTestItem, message); - } else { - throw new Error('Unable to create new child node for subtest'); - } - } else { - throw new Error('Parent test item not found'); - } - } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') { - // split on " " since the subtest ID has the parent test ID in the first part of the ID. - const parentTestCaseId = keyTemp.split(' ')[0]; - const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); - - // find the subtest's parent test item - if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); - if (subtestStats) { - subtestStats.passed += 1; - } else { - subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); - runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); - // clear since subtest items don't persist between runs - clearAllChildren(parentTestItem); - } - const subtestId = keyTemp; - const subTestItem = testController?.createTestItem(subtestId, subtestId); - // create a new test item for the subtest - if (subTestItem) { - parentTestItem.children.add(subTestItem); - runInstance.started(subTestItem); - runInstance.passed(subTestItem); - runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`)); - } else { - throw new Error('Unable to create new child node for subtest'); - } - } else { - throw new Error('Parent test item not found'); - } - } - } - } return Promise.resolve(); } @@ -286,8 +117,6 @@ export class WorkspaceTestAdapter { ): Promise { sendTelemetryEvent(EventName.UNITTEST_DISCOVERING, undefined, { tool: this.testProvider }); - const workspacePath = this.workspaceUri.fsPath; - // Discovery is expensive. If it is already running, use the existing promise. if (this.discovering) { return this.discovering.promise; @@ -296,14 +125,12 @@ export class WorkspaceTestAdapter { const deferred = createDeferred(); this.discovering = deferred; - let rawTestData; try { // ** execution factory only defined for new rewrite way if (executionFactory !== undefined) { - traceVerbose('executionFactory defined'); - rawTestData = await this.discoveryAdapter.discoverTests(this.workspaceUri, executionFactory); + await this.discoveryAdapter.discoverTests(this.workspaceUri, executionFactory); } else { - rawTestData = await this.discoveryAdapter.discoverTests(this.workspaceUri); + await this.discoveryAdapter.discoverTests(this.workspaceUri); } deferred.resolve(); } catch (ex) { @@ -324,114 +151,18 @@ export class WorkspaceTestAdapter { const errorNode = createErrorTestItem(testController, options); testController.items.add(errorNode); - deferred.reject(ex as Error); + return deferred.reject(ex as Error); } finally { // Discovery has finished running, we have the data, // we don't need the deferred promise anymore. this.discovering = undefined; } - if (!rawTestData) { - // No test data is available - return Promise.resolve(); - } - - // Check if there were any errors in the discovery process. - if (rawTestData.status === 'error') { - const testingErrorConst = - this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery; - const { errors } = rawTestData; - traceError(testingErrorConst, '\r\n', errors?.join('\r\n\r\n')); - let errorNode = testController.items.get(`DiscoveryError:${workspacePath}`); - const message = util.format( - `${testingErrorConst} ${Testing.seePythonOutput}\r\n`, - errors?.join('\r\n\r\n'), - ); - - if (errorNode === undefined) { - const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider); - errorNode = createErrorTestItem(testController, options); - testController.items.add(errorNode); - } - errorNode.error = message; - } else { - // Remove the error node if necessary, - // then parse and insert test data. - testController.items.delete(`DiscoveryError:${workspacePath}`); - - if (rawTestData.tests) { - // If the test root for this folder exists: Workspace refresh, update its children. - // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree. - populateTestTree(testController, rawTestData.tests, undefined, this, token); - } else { - // Delete everything from the test controller. - testController.items.replace([]); - } - } - sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: this.testProvider, failed: false }); return Promise.resolve(); } } -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} - -// had to switch the order of the original parameter since required param cannot follow optional. -function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - wstAdapter: WorkspaceTestAdapter, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - wstAdapter.runIdToTestItem.set(child.runID, testItem); - wstAdapter.runIdToVSid.set(child.runID, child.id_); - wstAdapter.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, wstAdapter, token); - } - } - }); -} - function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; return { diff --git a/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts index 12c79a23c7fd..0286235be1bf 100644 --- a/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestDiscoveryAdapter.unit.test.ts @@ -4,11 +4,17 @@ import * as assert from 'assert'; import { Uri } from 'vscode'; import * as typeMoq from 'typemoq'; +import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; import { PytestTestDiscoveryAdapter } from '../../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { DataReceivedEvent, ITestServer } from '../../../../client/testing/testController/common/types'; -import { IPythonExecutionFactory, IPythonExecutionService } from '../../../../client/common/process/types'; +import { ITestServer } from '../../../../client/testing/testController/common/types'; +import { + IPythonExecutionFactory, + IPythonExecutionService, + SpawnOptions, +} from '../../../../client/common/process/types'; import { createDeferred, Deferred } from '../../../../client/common/utils/async'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; suite('pytest test discovery adapter', () => { let testServer: typeMoq.IMock; @@ -18,27 +24,56 @@ suite('pytest test discovery adapter', () => { let execService: typeMoq.IMock; let deferred: Deferred; let outputChannel: typeMoq.IMock; + let portNum: number; + let uuid: string; + let expectedPath: string; + let uri: Uri; + let expectedExtraVariables: Record; setup(() => { + const mockExtensionRootDir = typeMoq.Mock.ofType(); + mockExtensionRootDir.setup((m) => m.toString()).returns(() => '/mocked/extension/root/dir'); + + // constants + portNum = 12345; + uuid = 'uuid123'; + expectedPath = path.join('/', 'my', 'test', 'path'); + uri = Uri.file(expectedPath); + const relativePathToPytest = 'pythonFiles'; + const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); + expectedExtraVariables = { + PYTHONPATH: fullPluginPath, + TEST_UUID: uuid, + TEST_PORT: portNum.toString(), + }; + + // set up test server testServer = typeMoq.Mock.ofType(); - testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer.setup((t) => t.getPort()).returns(() => portNum); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); testServer - .setup((t) => t.onDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns(() => ({ dispose: () => { /* no-body */ }, })); + + // set up config service configService = ({ getSettings: () => ({ testing: { pytestArgs: ['.'] }, }), } as unknown) as IConfigurationService; + + // set up exec factory execFactory = typeMoq.Mock.ofType(); - execService = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) .returns(() => Promise.resolve(execService.object)); + + // set up exec service + execService = typeMoq.Mock.ofType(); deferred = createDeferred(); execService .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) @@ -46,49 +81,51 @@ suite('pytest test discovery adapter', () => { deferred.resolve(); return Promise.resolve({ stdout: '{}' }); }); - execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); outputChannel = typeMoq.Mock.ofType(); }); - test('onDataReceivedHandler should parse only if known UUID', async () => { - const uri = Uri.file('/my/test/path/'); - const uuid = 'uuid123'; - const data = { status: 'success' }; - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); - const eventData: DataReceivedEvent = { - uuid, - data: JSON.stringify(data), - }; - + test('Discovery should call exec with correct basic args', async () => { adapter = new PytestTestDiscoveryAdapter(testServer.object, configService, outputChannel.object); - const promise = adapter.discoverTests(uri, execFactory.object); - // const promise = adapter.discoverTests(uri); - await deferred.promise; - adapter.onDataReceivedHandler(eventData); - const result = await promise; - assert.deepStrictEqual(result, data); + await adapter.discoverTests(uri, execFactory.object); + const expectedArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only', '.']; + + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.deepEqual(options.extraVariables, expectedExtraVariables); + assert.equal(options.cwd, expectedPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); }); - test('onDataReceivedHandler should not parse if it is unknown UUID', async () => { - const uri = Uri.file('/my/test/path/'); - const uuid = 'uuid456'; - let data = { status: 'error' }; - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); - const wrongUriEventData: DataReceivedEvent = { - uuid: 'incorrect-uuid456', - data: JSON.stringify(data), - }; - adapter = new PytestTestDiscoveryAdapter(testServer.object, configService, outputChannel.object); - const promise = adapter.discoverTests(uri, execFactory.object); - // const promise = adapter.discoverTests(uri); - adapter.onDataReceivedHandler(wrongUriEventData); + test('Test discovery correctly pulls pytest args from config service settings', async () => { + // set up a config service with different pytest args + const configServiceNew: IConfigurationService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.', 'abc', 'xyz'] }, + }), + } as unknown) as IConfigurationService; - data = { status: 'success' }; - const correctUriEventData: DataReceivedEvent = { - uuid, - data: JSON.stringify(data), - }; - adapter.onDataReceivedHandler(correctUriEventData); - const result = await promise; - assert.deepStrictEqual(result, data); + adapter = new PytestTestDiscoveryAdapter(testServer.object, configServiceNew, outputChannel.object); + await adapter.discoverTests(uri, execFactory.object); + const expectedArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only', '.', 'abc', 'xyz']; + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.deepEqual(options.extraVariables, expectedExtraVariables); + assert.equal(options.cwd, expectedPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); }); }); diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index ac6c6bd274a4..5e7a32ddc9e1 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -2,13 +2,18 @@ // // Copyright (c) Microsoft Corporation. All rights reserved. // // Licensed under the MIT License. // import * as assert from 'assert'; -// import { Uri } from 'vscode'; +// import { TestRun, Uri } from 'vscode'; // import * as typeMoq from 'typemoq'; -// import { IConfigurationService } from '../../../../client/common/types'; -// import { DataReceivedEvent, ITestServer } from '../../../../client/testing/testController/common/types'; -// import { IPythonExecutionFactory, IPythonExecutionService } from '../../../../client/common/process/types'; +// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +// import { ITestServer } from '../../../../client/testing/testController/common/types'; +// import { +// IPythonExecutionFactory, +// IPythonExecutionService, +// SpawnOptions, +// } from '../../../../client/common/process/types'; // import { createDeferred, Deferred } from '../../../../client/common/utils/async'; // import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; +// import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; // suite('pytest test execution adapter', () => { // let testServer: typeMoq.IMock; @@ -17,11 +22,12 @@ // let adapter: PytestTestExecutionAdapter; // let execService: typeMoq.IMock; // let deferred: Deferred; +// let debugLauncher: typeMoq.IMock; // setup(() => { // testServer = typeMoq.Mock.ofType(); // testServer.setup((t) => t.getPort()).returns(() => 12345); // testServer -// .setup((t) => t.onDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) // .returns(() => ({ // dispose: () => { // /* no-body */ @@ -35,6 +41,7 @@ // } as unknown) as IConfigurationService; // execFactory = typeMoq.Mock.ofType(); // execService = typeMoq.Mock.ofType(); +// debugLauncher = typeMoq.Mock.ofType(); // execFactory // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) // .returns(() => Promise.resolve(execService.object)); @@ -45,46 +52,90 @@ // deferred.resolve(); // return Promise.resolve({ stdout: '{}' }); // }); +// debugLauncher +// .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => { +// deferred.resolve(); +// return Promise.resolve(); +// }); // execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); // execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); +// debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); // }); -// test('onDataReceivedHandler should parse only if known UUID', async () => { +// test('pytest execution called with correct args', async () => { // const uri = Uri.file('/my/test/path/'); // const uuid = 'uuid123'; -// const data = { status: 'success' }; +// // const data = { status: 'success' }; +// testServer +// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => ({ +// dispose: () => { +// /* no-body */ +// }, +// })); // testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const eventData: DataReceivedEvent = { -// uuid, -// data: JSON.stringify(data), -// }; +// const outputChannel = typeMoq.Mock.ofType(); +// const testRun = typeMoq.Mock.ofType(); +// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); +// await adapter.runTests(uri, [], false, testRun.object, execFactory.object); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService); -// const promise = adapter.runTests(uri, [], false); -// await deferred.promise; -// adapter.onDataReceivedHandler(eventData); -// const result = await promise; -// assert.deepStrictEqual(result, data); +// const expectedArgs = [ +// '/Users/eleanorboyd/vscode-python/pythonFiles/vscode_pytest/run_pytest_script.py', +// '--rootdir', +// '/my/test/path/', +// ]; +// const expectedExtraVariables = { +// PYTHONPATH: '/Users/eleanorboyd/vscode-python/pythonFiles', +// TEST_UUID: 'uuid123', +// TEST_PORT: '12345', +// }; +// execService.verify( +// (x) => +// x.exec( +// expectedArgs, +// typeMoq.It.is((options) => { +// assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); +// assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); +// assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); +// assert.strictEqual(typeof options.extraVariables?.RUN_TEST_IDS_PORT, 'string'); +// assert.equal(options.cwd, uri.fsPath); +// assert.equal(options.throwOnStdErr, true); +// return true; +// }), +// ), +// typeMoq.Times.once(), +// ); // }); -// test('onDataReceivedHandler should not parse if it is unknown UUID', async () => { +// test('Debug launched correctly for pytest', async () => { // const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid456'; -// let data = { status: 'error' }; +// const uuid = 'uuid123'; +// testServer +// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => ({ +// dispose: () => { +// /* no-body */ +// }, +// })); // testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const wrongUriEventData: DataReceivedEvent = { -// uuid: 'incorrect-uuid456', -// data: JSON.stringify(data), -// }; -// adapter = new PytestTestExecutionAdapter(testServer.object, configService); -// const promise = adapter.runTests(uri, [], false); -// adapter.onDataReceivedHandler(wrongUriEventData); - -// data = { status: 'success' }; -// const correctUriEventData: DataReceivedEvent = { -// uuid, -// data: JSON.stringify(data), -// }; -// adapter.onDataReceivedHandler(correctUriEventData); -// const result = await promise; -// assert.deepStrictEqual(result, data); +// const outputChannel = typeMoq.Mock.ofType(); +// const testRun = typeMoq.Mock.ofType(); +// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); +// await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); +// debugLauncher.verify( +// (x) => +// x.launchDebugger( +// typeMoq.It.is((launchOptions) => { +// assert.equal(launchOptions.cwd, uri.fsPath); +// assert.deepEqual(launchOptions.args, ['--rootdir', '/my/test/path/', '--capture', 'no']); +// assert.equal(launchOptions.testProvider, 'pytest'); +// assert.equal(launchOptions.pytestPort, '12345'); +// assert.equal(launchOptions.pytestUUID, 'uuid123'); +// assert.strictEqual(typeof launchOptions.runTestIdsPort, 'string'); +// return true; +// }), +// typeMoq.It.isAny(), +// ), +// typeMoq.Times.once(), +// ); // }); // }); diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index d7b3a242ee9a..38b71992aefb 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -1,158 +1,349 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import * as assert from 'assert'; -import * as net from 'net'; -import * as sinon from 'sinon'; -import * as crypto from 'crypto'; -import { OutputChannel, Uri } from 'vscode'; -import { IPythonExecutionFactory, IPythonExecutionService } from '../../../client/common/process/types'; -import { PythonTestServer } from '../../../client/testing/testController/common/server'; -import { ITestDebugLauncher } from '../../../client/testing/common/types'; -import { createDeferred } from '../../../client/common/utils/async'; - -suite('Python Test Server', () => { - const fakeUuid = 'fake-uuid'; - - let stubExecutionFactory: IPythonExecutionFactory; - let stubExecutionService: IPythonExecutionService; - let server: PythonTestServer; - let sandbox: sinon.SinonSandbox; - let execArgs: string[]; - let v4Stub: sinon.SinonStub; - let debugLauncher: ITestDebugLauncher; - - setup(() => { - sandbox = sinon.createSandbox(); - v4Stub = sandbox.stub(crypto, 'randomUUID'); - - v4Stub.returns(fakeUuid); - stubExecutionService = ({ - exec: (args: string[]) => { - execArgs = args; - return Promise.resolve({ stdout: '', stderr: '' }); - }, - } as unknown) as IPythonExecutionService; - - stubExecutionFactory = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService), - } as unknown) as IPythonExecutionFactory; - }); - - teardown(() => { - sandbox.restore(); - execArgs = []; - server.dispose(); - }); - - test('sendCommand should add the port to the command being sent', async () => { - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - await server.sendCommand(options); - const port = server.getPort(); - - assert.deepStrictEqual(execArgs, ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']); - }); - - test('sendCommand should write to an output channel if it is provided as an option', async () => { - const output: string[] = []; - const outChannel = { - appendLine: (str: string) => { - output.push(str); - }, - } as OutputChannel; - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - outChannel, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - await server.sendCommand(options); - - const port = server.getPort(); - const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); - - assert.deepStrictEqual(output, [expected]); - }); - - test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { - let eventData: { status: string; errors: string[] }; - stubExecutionService = ({ - exec: () => { - throw new Error('Failed to execute'); - }, - } as unknown) as IPythonExecutionService; - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - server.onDataReceived(({ data }) => { - eventData = JSON.parse(data); - }); - - await server.sendCommand(options); - - assert.deepStrictEqual(eventData!.status, 'error'); - assert.deepStrictEqual(eventData!.errors, ['Failed to execute']); - }); - - test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const deferred = createDeferred(); - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - stubExecutionService = ({ - exec: async () => { - client.connect(server.getPort()); - return Promise.resolve({ stdout: '', stderr: '' }); - }, - } as unknown) as IPythonExecutionService; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('malformed data'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - await server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); -}); +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. + +// import * as assert from 'assert'; +// import * as net from 'net'; +// import * as sinon from 'sinon'; +// import * as crypto from 'crypto'; +// import { Uri } from 'vscode'; +// import { IPythonExecutionFactory, IPythonExecutionService } from '../../../client/common/process/types'; +// import { PythonTestServer } from '../../../client/testing/testController/common/server'; +// import { ITestDebugLauncher } from '../../../client/testing/common/types'; +// import { createDeferred } from '../../../client/common/utils/async'; + +// suite('Python Test Server', () => { +// const fakeUuid = 'fake-uuid'; + +// let stubExecutionFactory: IPythonExecutionFactory; +// let stubExecutionService: IPythonExecutionService; +// let server: PythonTestServer; +// let sandbox: sinon.SinonSandbox; +// let execArgs: string[]; +// let v4Stub: sinon.SinonStub; +// let debugLauncher: ITestDebugLauncher; + +// setup(() => { +// sandbox = sinon.createSandbox(); +// v4Stub = sandbox.stub(crypto, 'randomUUID'); + +// v4Stub.returns(fakeUuid); +// stubExecutionService = ({ +// exec: (args: string[]) => { +// execArgs = args; +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// stubExecutionFactory = ({ +// createActivatedEnvironment: () => Promise.resolve(stubExecutionService), +// } as unknown) as IPythonExecutionFactory; +// }); + +// teardown(() => { +// sandbox.restore(); +// execArgs = []; +// server.dispose(); +// }); + +// // test('sendCommand should add the port to the command being sent', async () => { +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // await server.sendCommand(options); +// // const port = server.getPort(); + +// // assert.deepStrictEqual(execArgs, ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']); +// // }); + +// // test('sendCommand should write to an output channel if it is provided as an option', async () => { +// // const output: string[] = []; +// // const outChannel = { +// // appendLine: (str: string) => { +// // output.push(str); +// // }, +// // } as OutputChannel; +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // outChannel, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // await server.sendCommand(options); + +// // const port = server.getPort(); +// // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); + +// // assert.deepStrictEqual(output, [expected]); +// // }); + +// // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { +// // let eventData: { status: string; errors: string[] }; +// // stubExecutionService = ({ +// // exec: () => { +// // throw new Error('Failed to execute'); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // server.onDataReceived(({ data }) => { +// // eventData = JSON.parse(data); +// // }); + +// // await server.sendCommand(options); + +// // assert.deepStrictEqual(eventData!.status, 'error'); +// // assert.deepStrictEqual(eventData!.errors, ['Failed to execute']); +// // }); + +// // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('malformed data'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // test('If the server doesnt recognize the UUID it should ignore it', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"Request-uuid": "unknown-uuid"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // required to have "tests" or "results" +// // the heading length not being equal and yes being equal +// // multiple payloads +// // test('Error if payload does not have a content length header', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const deferred = createDeferred(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: fakeUuid, +// // }; + +// // stubExecutionService = ({ +// // exec: async () => { +// // client.connect(server.getPort()); +// // return Promise.resolve({ stdout: '', stderr: '' }); +// // }, +// // } as unknown) as IPythonExecutionService; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"not content length": "5"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // await server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// const testData = [ +// { +// testName: 'fires discovery correctly on test payload', +// payload: `Content-Length: 52 +// Content-Type: application/json +// Request-uuid: UUID_HERE + +// {"cwd": "path", "status": "success", "tests": "xyz"}`, +// expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', +// }, +// // Add more test data as needed +// ]; + +// testData.forEach(({ testName, payload, expectedResult }) => { +// test(`test: ${testName}`, async () => { +// // Your test logic here +// let eventData: string | undefined; +// const client = new net.Socket(); +// const deferred = createDeferred(); + +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid: fakeUuid, +// }; + +// stubExecutionService = ({ +// exec: async () => { +// client.connect(server.getPort()); +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// await server.serverReady(); +// const uuid = server.createUUID(); +// payload = payload.replace('UUID_HERE', uuid); +// server.onDiscoveryDataReceived(({ data }) => { +// eventData = data; +// deferred.resolve(); +// }); + +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// client.write(payload); +// client.end(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// await server.sendCommand(options); +// await deferred.promise; +// assert.deepStrictEqual(eventData, expectedResult); +// }); +// }); + +// test('Calls run resolver if the result header is in the payload', async () => { +// let eventData: string | undefined; +// const client = new net.Socket(); +// const deferred = createDeferred(); + +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid: fakeUuid, +// }; + +// stubExecutionService = ({ +// exec: async () => { +// client.connect(server.getPort()); +// return Promise.resolve({ stdout: '', stderr: '' }); +// }, +// } as unknown) as IPythonExecutionService; + +// server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// await server.serverReady(); +// const uuid = server.createUUID(); +// server.onRunDataReceived(({ data }) => { +// eventData = data; +// deferred.resolve(); +// }); + +// const payload = `Content-Length: 87 +// Content-Type: application/json +// Request-uuid: ${uuid} + +// {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// client.write(payload); +// client.end(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// await server.sendCommand(options); +// await deferred.promise; +// console.log('event data', eventData); +// const expectedResult = +// '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; +// assert.deepStrictEqual(eventData, expectedResult); +// }); +// }); diff --git a/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts index 3d3521291f74..ef21655e93e4 100644 --- a/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts @@ -23,7 +23,7 @@ suite('Unittest test discovery adapter', () => { outputChannel = typemoq.Mock.ofType(); }); - test('discoverTests should send the discovery command to the test server', async () => { + test('DiscoverTests should send the discovery command to the test server with the correct args', async () => { let options: TestCommandOptions | undefined; const stubTestServer = ({ @@ -32,7 +32,7 @@ suite('Unittest test discovery adapter', () => { options = opt; return Promise.resolve(); }, - onDataReceived: () => { + onDiscoveryDataReceived: () => { // no body }, createUUID: () => '123456789', @@ -47,61 +47,11 @@ suite('Unittest test discovery adapter', () => { assert.deepStrictEqual(options, { workspaceFolder: uri, cwd: uri.fsPath, - command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, + command: { + script, + args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'], + }, uuid: '123456789', }); }); - - test("onDataReceivedHandler should parse the data if the cwd from the payload matches the test adapter's cwd", async () => { - const stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - createUUID: () => '123456789', - } as unknown) as ITestServer; - - const uri = Uri.file('/foo/bar'); - const data = { status: 'success' }; - const uuid = '123456789'; - const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - const promise = adapter.discoverTests(uri); - - adapter.onDataReceivedHandler({ uuid, data: JSON.stringify(data) }); - - const result = await promise; - - assert.deepStrictEqual(result, data); - }); - - test("onDataReceivedHandler should ignore the data if the cwd from the payload does not match the test adapter's cwd", async () => { - const correctUuid = '123456789'; - const incorrectUuid = '987654321'; - const stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - createUUID: () => correctUuid, - } as unknown) as ITestServer; - - const uri = Uri.file('/foo/bar'); - - const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - const promise = adapter.discoverTests(uri); - - const data = { status: 'success' }; - adapter.onDataReceivedHandler({ uuid: incorrectUuid, data: JSON.stringify(data) }); - - const nextData = { status: 'error' }; - adapter.onDataReceivedHandler({ uuid: correctUuid, data: JSON.stringify(nextData) }); - - const result = await promise; - - assert.deepStrictEqual(result, nextData); - }); }); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 539647aece9f..42e38d200546 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,246 +1,267 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import * as assert from 'assert'; -import * as sinon from 'sinon'; -import * as typemoq from 'typemoq'; - -import { TestController, TestItem, Uri } from 'vscode'; -import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; -import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 -import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; -import * as Telemetry from '../../../client/telemetry'; -import { EventName } from '../../../client/telemetry/constants'; -import { ITestServer } from '../../../client/testing/testController/common/types'; - -suite('Workspace test adapter', () => { - suite('Test discovery', () => { - let stubTestServer: ITestServer; - let stubConfigSettings: IConfigurationService; - - let discoverTestsStub: sinon.SinonStub; - let sendTelemetryStub: sinon.SinonStub; - let outputChannel: typemoq.IMock; - - let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - - // Stubbed test controller (see comment around L.40) - let testController: TestController; - let log: string[] = []; - - const sandbox = sinon.createSandbox(); - - setup(() => { - stubConfigSettings = ({ - getSettings: () => ({ - testing: { unittestArgs: ['--foo'] }, - }), - } as unknown) as IConfigurationService; - - stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - } as unknown) as ITestServer; - - // For some reason the 'tests' namespace in vscode returns undefined. - // While I figure out how to expose to the tests, they will run - // against a stub test controller and stub test items. - const testItem = ({ - canResolveChildren: false, - tags: [], - children: { - add: () => { - // empty - }, - }, - } as unknown) as TestItem; - - testController = ({ - items: { - get: () => { - log.push('get'); - }, - add: () => { - log.push('add'); - }, - replace: () => { - log.push('replace'); - }, - delete: () => { - log.push('delete'); - }, - }, - createTestItem: () => { - log.push('createTestItem'); - return testItem; - }, - dispose: () => { - // empty - }, - } as unknown) as TestController; - - // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - - const mockSendTelemetryEvent = ( - eventName: EventName, - _: number | Record | undefined, - properties: unknown, - ) => { - telemetryEvent.push({ - eventName, - properties: properties as Record, - }); - }; - - discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); - sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); - outputChannel = typemoq.Mock.ofType(); - }); - - teardown(() => { - telemetryEvent = []; - log = []; - testController.dispose(); - sandbox.restore(); - }); - - test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { - discoverTestsStub.resolves(); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { - discoverTestsStub.callsFake( - async () => - new Promise((resolve) => { - setTimeout(() => { - // Simulate time taken by discovery. - resolve(); - }, 2000); - }), - ); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - // Try running discovery twice - const one = workspaceTestAdapter.discoverTests(testController); - const two = workspaceTestAdapter.discoverTests(testController); - - Promise.all([one, two]); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { - discoverTestsStub.resolves({ status: 'success' }); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); - - const lastEvent = telemetryEvent[1]; - assert.strictEqual(lastEvent.properties.failed, false); - }); - - test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { - discoverTestsStub.rejects(new Error('foo')); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - ); - - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); - - const lastEvent = telemetryEvent[1]; - assert.ok(lastEvent.properties.failed); - - assert.deepStrictEqual(log, ['createTestItem', 'add']); - }); - - /** - * TODO To test: - * - successful discovery but no data: delete everything from the test controller - * - successful discovery with error status: add error node to tree - * - single root: populate tree if there's no root node - * - single root: update tree if there's a root node - * - single root: delete tree if there are no tests in the test data - * - multiroot: update the correct folders - */ - }); -}); +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. + +// import * as assert from 'assert'; +// import * as sinon from 'sinon'; +// import * as typemoq from 'typemoq'; + +// import { TestController, TestItem, Uri } from 'vscode'; +// import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 +// import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +// import * as Telemetry from '../../../client/telemetry'; +// import { EventName } from '../../../client/telemetry/constants'; +// import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; + +// suite('Workspace test adapter', () => { +// suite('Test discovery', () => { +// let stubTestServer: ITestServer; +// let stubConfigSettings: IConfigurationService; +// let stubResultResolver: ITestResultResolver; + +// let discoverTestsStub: sinon.SinonStub; +// let sendTelemetryStub: sinon.SinonStub; +// let outputChannel: typemoq.IMock; + +// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + +// // Stubbed test controller (see comment around L.40) +// let testController: TestController; +// let log: string[] = []; + +// const sandbox = sinon.createSandbox(); + +// setup(() => { +// stubConfigSettings = ({ +// getSettings: () => ({ +// testing: { unittestArgs: ['--foo'] }, +// }), +// } as unknown) as IConfigurationService; + +// stubTestServer = ({ +// sendCommand(): Promise { +// return Promise.resolve(); +// }, +// onDataReceived: () => { +// // no body +// }, +// } as unknown) as ITestServer; + +// stubResultResolver = ({ +// resolveDiscovery: () => { +// // no body +// }, +// resolveExecution: () => { +// // no body +// }, +// vsIdToRunId: { +// get: sinon.stub().returns('expectedRunId'), +// }, +// } as unknown) as ITestResultResolver; + +// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); +// // const expectedRunId = 'expectedRunId'; +// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + +// // For some reason the 'tests' namespace in vscode returns undefined. +// // While I figure out how to expose to the tests, they will run +// // against a stub test controller and stub test items. +// const testItem = ({ +// canResolveChildren: false, +// tags: [], +// children: { +// add: () => { +// // empty +// }, +// }, +// } as unknown) as TestItem; + +// testController = ({ +// items: { +// get: () => { +// log.push('get'); +// }, +// add: () => { +// log.push('add'); +// }, +// replace: () => { +// log.push('replace'); +// }, +// delete: () => { +// log.push('delete'); +// }, +// }, +// createTestItem: () => { +// log.push('createTestItem'); +// return testItem; +// }, +// dispose: () => { +// // empty +// }, +// } as unknown) as TestController; + +// // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + +// const mockSendTelemetryEvent = ( +// eventName: EventName, +// _: number | Record | undefined, +// properties: unknown, +// ) => { +// telemetryEvent.push({ +// eventName, +// properties: properties as Record, +// }); +// }; + +// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); +// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); +// outputChannel = typemoq.Mock.ofType(); +// }); + +// teardown(() => { +// telemetryEvent = []; +// log = []; +// testController.dispose(); +// sandbox.restore(); +// }); + +// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { +// discoverTestsStub.resolves(); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledOnce(discoverTestsStub); +// }); + +// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { +// discoverTestsStub.callsFake( +// async () => +// new Promise((resolve) => { +// setTimeout(() => { +// // Simulate time taken by discovery. +// resolve(); +// }, 2000); +// }), +// ); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// // Try running discovery twice +// const one = workspaceTestAdapter.discoverTests(testController); +// const two = workspaceTestAdapter.discoverTests(testController); + +// Promise.all([one, two]); + +// sinon.assert.calledOnce(discoverTestsStub); +// }); + +// test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { +// discoverTestsStub.resolves({ status: 'success' }); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); + +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); +// assert.strictEqual(telemetryEvent.length, 2); + +// const lastEvent = telemetryEvent[1]; +// assert.strictEqual(lastEvent.properties.failed, false); +// }); + +// test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { +// discoverTestsStub.rejects(new Error('foo')); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); + +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); + +// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); +// assert.strictEqual(telemetryEvent.length, 2); + +// const lastEvent = telemetryEvent[1]; +// assert.ok(lastEvent.properties.failed); + +// assert.deepStrictEqual(log, ['createTestItem', 'add']); +// }); + +// /** +// * TODO To test: +// * - successful discovery but no data: delete everything from the test controller +// * - successful discovery with error status: add error node to tree +// * - single root: populate tree if there's no root node +// * - single root: update tree if there's a root node +// * - single root: delete tree if there are no tests in the test data +// * - multiroot: update the correct folders +// */ +// }); +// }); From 0779d2eb67925c245c12401170273b62f5e10c6c Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 19:00:03 -0700 Subject: [PATCH 08/34] Move testid server (#21400) Created a testIdServer to handle the starting up of the testIdServer for both unittest and pytest. This extracts the method to be in another file and easy to test. --- .../testing/testController/common/utils.ts | 2 +- .../pytest/pytestExecutionAdapter.ts | 36 ++----------------- .../unittest/testExecutionAdapter.ts | 35 +++--------------- 3 files changed, 8 insertions(+), 65 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 1bf31e80e11a..9d426dc60ffa 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -62,7 +62,7 @@ export function pythonTestAdapterRewriteEnabled(serviceContainer: IServiceContai return experiment.inExperimentSync(EnableTestAdapterRewrite.experiment); } -export const startServer = (testIds: string): Promise => +export const startTestIdServer = (testIds: string[]): Promise => new Promise((resolve, reject) => { const server = net.createServer((socket: net.Socket) => { // Convert the test_ids array to JSON diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 6d63b49d007f..9c8a42ae097f 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -3,7 +3,6 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; -import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { createDeferred, Deferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; @@ -23,6 +22,7 @@ import { removePositionalFoldersAndFiles } from './arguments'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { PYTEST_PROVIDER } from '../../common/constants'; import { EXTENSION_ROOT_DIR } from '../../../common/constants'; +import { startTestIdServer } from '../common/utils'; // eslint-disable-next-line @typescript-eslint/no-explicit-any // (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; @@ -116,40 +116,10 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { if (debugBool && !testArgs.some((a) => a.startsWith('--capture') || a === '-s')) { testArgs.push('--capture', 'no'); } - - // create payload with testIds to send to run pytest script - const testData = JSON.stringify(testIds); - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; - traceLog(`Running pytest execution for the following test ids: ${testIds}`); + traceLog(`Running PYTEST execution for the following test ids: ${testIds}`); let pytestRunTestIdsPort: string | undefined; - const startServer = (): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - socket.on('end', () => { - traceVerbose('Client disconnected for pytest test ids server'); - }); - }); - - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceVerbose(`Server listening on port ${port} for pytest test ids server`); - resolve(port); - }); - - server.on('error', (error: Error) => { - traceError('Error starting server for pytest test ids server:', error); - reject(error); - }); - server.on('connection', (socket: net.Socket) => { - socket.write(payload); - traceVerbose('payload sent for pytest execution', payload); - }); - }); - - // Start the server and wait until it is listening - await startServer() + await startTestIdServer(testIds) .then((assignedPort) => { traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); pytestRunTestIdsPort = assignedPort.toString(); diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index b18b5945c6b0..24e9cd301e30 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -3,9 +3,8 @@ import * as path from 'path'; import { TestRun, Uri } from 'vscode'; -import * as net from 'net'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred, Deferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -17,6 +16,7 @@ import { TestExecutionCommand, } from '../common/types'; import { traceLog, traceError } from '../../../logging'; +import { startTestIdServer } from '../common/utils'; /** * Wrapper Class for unittest test execution. This is where we call `runTestCommand`? @@ -75,37 +75,10 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const deferred = createDeferred(); this.promiseMap.set(uuid, deferred); - // create payload with testIds to send to run pytest script - const testData = JSON.stringify(testIds); - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; + traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); let runTestIdsPort: string | undefined; - const startServer = (): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - socket.on('end', () => { - traceLog('Client disconnected'); - }); - }); - - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceLog(`Server listening on port ${port}`); - resolve(port); - }); - - server.on('error', (error: Error) => { - reject(error); - }); - server.on('connection', (socket: net.Socket) => { - socket.write(payload); - traceLog('payload sent', payload); - }); - }); - - // Start the server and wait until it is listening - await startServer() + await startTestIdServer(testIds) .then((assignedPort) => { traceLog(`Server started and listening on port ${assignedPort}`); runTestIdsPort = assignedPort.toString(); From 20cf981a6ff8dd185962e3a9b70c6ca2be8bb8e7 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 13 Jun 2023 16:30:37 -0700 Subject: [PATCH 09/34] Execution adapter tests (#21421) adds test coverage for `testExecutionAdapter.ts` and `pytestExecutionAdapter.ts`. --- .../testing/testController/common/utils.ts | 64 ++-- .../pytest/pytestExecutionAdapter.ts | 16 +- .../unittest/testExecutionAdapter.ts | 16 +- .../pytestExecutionAdapter.unit.test.ts | 289 +++++++++--------- .../testExecutionAdapter.unit.test.ts | 182 ++++------- 5 files changed, 260 insertions(+), 307 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 9d426dc60ffa..232aaccd5341 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as net from 'net'; -import { traceLog } from '../../../logging'; +import { traceError, traceLog, traceVerbose } from '../../../logging'; import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; @@ -62,38 +62,52 @@ export function pythonTestAdapterRewriteEnabled(serviceContainer: IServiceContai return experiment.inExperimentSync(EnableTestAdapterRewrite.experiment); } -export const startTestIdServer = (testIds: string[]): Promise => - new Promise((resolve, reject) => { - const server = net.createServer((socket: net.Socket) => { - // Convert the test_ids array to JSON - const testData = JSON.stringify(testIds); +export async function startTestIdServer(testIds: string[]): Promise { + const startServer = (): Promise => + new Promise((resolve, reject) => { + const server = net.createServer((socket: net.Socket) => { + // Convert the test_ids array to JSON + const testData = JSON.stringify(testIds); - // Create the headers - const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; + // Create the headers + const headers = [`Content-Length: ${Buffer.byteLength(testData)}`, 'Content-Type: application/json']; - // Create the payload by concatenating the headers and the test data - const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; + // Create the payload by concatenating the headers and the test data + const payload = `${headers.join('\r\n')}\r\n\r\n${testData}`; - // Send the payload to the socket - socket.write(payload); + // Send the payload to the socket + socket.write(payload); - // Handle socket events - socket.on('data', (data) => { - traceLog('Received data:', data.toString()); + // Handle socket events + socket.on('data', (data) => { + traceLog('Received data:', data.toString()); + }); + + socket.on('end', () => { + traceLog('Client disconnected'); + }); }); - socket.on('end', () => { - traceLog('Client disconnected'); + server.listen(0, () => { + const { port } = server.address() as net.AddressInfo; + traceLog(`Server listening on port ${port}`); + resolve(port); }); - }); - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - traceLog(`Server listening on port ${port}`); - resolve(port); + server.on('error', (error: Error) => { + reject(error); + }); }); - server.on('error', (error: Error) => { - reject(error); + // Start the server and wait until it is listening + await startServer() + .then((assignedPort) => { + traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); + return assignedPort; + // if (spawnOptions.extraVariables) spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; + }) + .catch((error) => { + traceError('Error starting server for pytest test ids server:', error); }); - }); + return 0; +} diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 9c8a42ae097f..1bf032b9b594 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -118,17 +118,9 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { } traceLog(`Running PYTEST execution for the following test ids: ${testIds}`); - let pytestRunTestIdsPort: string | undefined; - await startTestIdServer(testIds) - .then((assignedPort) => { - traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); - pytestRunTestIdsPort = assignedPort.toString(); - if (spawnOptions.extraVariables) - spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; - }) - .catch((error) => { - traceError('Error starting server for pytest test ids server:', error); - }); + const pytestRunTestIdsPort = await startTestIdServer(testIds); + if (spawnOptions.extraVariables) + spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort.toString(); if (debugBool) { const pytestPort = this.testServer.getPort().toString(); @@ -140,7 +132,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { testProvider: PYTEST_PROVIDER, pytestPort, pytestUUID, - runTestIdsPort: pytestRunTestIdsPort, + runTestIdsPort: pytestRunTestIdsPort.toString(), }; traceInfo(`Running DEBUG pytest with arguments: ${testArgs.join(' ')}\r\n`); await debugLauncher!.launchDebugger(launchOptions, () => { diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index 24e9cd301e30..ca88d3871706 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -15,7 +15,7 @@ import { TestCommandOptions, TestExecutionCommand, } from '../common/types'; -import { traceLog, traceError } from '../../../logging'; +import { traceLog } from '../../../logging'; import { startTestIdServer } from '../common/utils'; /** @@ -77,19 +77,9 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { this.promiseMap.set(uuid, deferred); traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); - let runTestIdsPort: string | undefined; - await startTestIdServer(testIds) - .then((assignedPort) => { - traceLog(`Server started and listening on port ${assignedPort}`); - runTestIdsPort = assignedPort.toString(); - // Send test command to server. - // Server fire onDataReceived event once it gets response. - }) - .catch((error) => { - traceError('Error starting server:', error); - }); + const runTestIdsPort = await startTestIdServer(testIds); - await this.testServer.sendCommand(options, runTestIdsPort, () => { + await this.testServer.sendCommand(options, runTestIdsPort.toString(), () => { // disposable.dispose(); deferred.resolve(); }); diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 5e7a32ddc9e1..f60429d38e5b 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -1,141 +1,152 @@ -// /* eslint-disable @typescript-eslint/no-explicit-any */ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. -// import * as assert from 'assert'; -// import { TestRun, Uri } from 'vscode'; -// import * as typeMoq from 'typemoq'; -// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; -// import { ITestServer } from '../../../../client/testing/testController/common/types'; -// import { -// IPythonExecutionFactory, -// IPythonExecutionService, -// SpawnOptions, -// } from '../../../../client/common/process/types'; -// import { createDeferred, Deferred } from '../../../../client/common/utils/async'; -// import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; -// import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import * as assert from 'assert'; +import { TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as sinon from 'sinon'; +import * as path from 'path'; +import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +import { ITestServer } from '../../../../client/testing/testController/common/types'; +import { + IPythonExecutionFactory, + IPythonExecutionService, + SpawnOptions, +} from '../../../../client/common/process/types'; +import { createDeferred, Deferred } from '../../../../client/common/utils/async'; +import { PytestTestExecutionAdapter } from '../../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/common/types'; +import * as util from '../../../../client/testing/testController/common/utils'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; -// suite('pytest test execution adapter', () => { -// let testServer: typeMoq.IMock; -// let configService: IConfigurationService; -// let execFactory = typeMoq.Mock.ofType(); -// let adapter: PytestTestExecutionAdapter; -// let execService: typeMoq.IMock; -// let deferred: Deferred; -// let debugLauncher: typeMoq.IMock; -// setup(() => { -// testServer = typeMoq.Mock.ofType(); -// testServer.setup((t) => t.getPort()).returns(() => 12345); -// testServer -// .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// configService = ({ -// getSettings: () => ({ -// testing: { pytestArgs: ['.'] }, -// }), -// isTestExecution: () => false, -// } as unknown) as IConfigurationService; -// execFactory = typeMoq.Mock.ofType(); -// execService = typeMoq.Mock.ofType(); -// debugLauncher = typeMoq.Mock.ofType(); -// execFactory -// .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) -// .returns(() => Promise.resolve(execService.object)); -// deferred = createDeferred(); -// execService -// .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => { -// deferred.resolve(); -// return Promise.resolve({ stdout: '{}' }); -// }); -// debugLauncher -// .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => { -// deferred.resolve(); -// return Promise.resolve(); -// }); -// execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// }); -// test('pytest execution called with correct args', async () => { -// const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid123'; -// // const data = { status: 'success' }; -// testServer -// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const outputChannel = typeMoq.Mock.ofType(); -// const testRun = typeMoq.Mock.ofType(); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); -// await adapter.runTests(uri, [], false, testRun.object, execFactory.object); +suite('pytest test execution adapter', () => { + let testServer: typeMoq.IMock; + let configService: IConfigurationService; + let execFactory = typeMoq.Mock.ofType(); + let adapter: PytestTestExecutionAdapter; + let execService: typeMoq.IMock; + let deferred: Deferred; + let debugLauncher: typeMoq.IMock; + (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; + let myTestPath: string; -// const expectedArgs = [ -// '/Users/eleanorboyd/vscode-python/pythonFiles/vscode_pytest/run_pytest_script.py', -// '--rootdir', -// '/my/test/path/', -// ]; -// const expectedExtraVariables = { -// PYTHONPATH: '/Users/eleanorboyd/vscode-python/pythonFiles', -// TEST_UUID: 'uuid123', -// TEST_PORT: '12345', -// }; -// execService.verify( -// (x) => -// x.exec( -// expectedArgs, -// typeMoq.It.is((options) => { -// assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); -// assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); -// assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); -// assert.strictEqual(typeof options.extraVariables?.RUN_TEST_IDS_PORT, 'string'); -// assert.equal(options.cwd, uri.fsPath); -// assert.equal(options.throwOnStdErr, true); -// return true; -// }), -// ), -// typeMoq.Times.once(), -// ); -// }); -// test('Debug launched correctly for pytest', async () => { -// const uri = Uri.file('/my/test/path/'); -// const uuid = 'uuid123'; -// testServer -// .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// dispose: () => { -// /* no-body */ -// }, -// })); -// testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); -// const outputChannel = typeMoq.Mock.ofType(); -// const testRun = typeMoq.Mock.ofType(); -// adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); -// await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); -// debugLauncher.verify( -// (x) => -// x.launchDebugger( -// typeMoq.It.is((launchOptions) => { -// assert.equal(launchOptions.cwd, uri.fsPath); -// assert.deepEqual(launchOptions.args, ['--rootdir', '/my/test/path/', '--capture', 'no']); -// assert.equal(launchOptions.testProvider, 'pytest'); -// assert.equal(launchOptions.pytestPort, '12345'); -// assert.equal(launchOptions.pytestUUID, 'uuid123'); -// assert.strictEqual(typeof launchOptions.runTestIdsPort, 'string'); -// return true; -// }), -// typeMoq.It.isAny(), -// ), -// typeMoq.Times.once(), -// ); -// }); -// }); + setup(() => { + testServer = typeMoq.Mock.ofType(); + testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + configService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.'] }, + }), + isTestExecution: () => false, + } as unknown) as IConfigurationService; + execFactory = typeMoq.Mock.ofType(); + execService = typeMoq.Mock.ofType(); + debugLauncher = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execService.object)); + deferred = createDeferred(); + execService + .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve({ stdout: '{}' }); + }); + debugLauncher + .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve(); + }); + sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + + execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + myTestPath = path.join('/', 'my', 'test', 'path', '/'); + }); + teardown(() => { + sinon.restore(); + }); + test('pytest execution called with correct args', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + await adapter.runTests(uri, [], false, testRun.object, execFactory.object); + + const pathToPythonFiles = path.join(EXTENSION_ROOT_DIR, 'pythonFiles'); + const pathToPythonScript = path.join(pathToPythonFiles, 'vscode_pytest', 'run_pytest_script.py'); + const expectedArgs = [pathToPythonScript, '--rootdir', myTestPath]; + const expectedExtraVariables = { + PYTHONPATH: pathToPythonFiles, + TEST_UUID: 'uuid123', + TEST_PORT: '12345', + }; + // execService.verify((x) => x.exec(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); + execService.verify( + (x) => + x.exec( + expectedArgs, + typeMoq.It.is((options) => { + assert.equal(options.extraVariables?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); + assert.equal(options.extraVariables?.TEST_UUID, expectedExtraVariables.TEST_UUID); + assert.equal(options.extraVariables?.TEST_PORT, expectedExtraVariables.TEST_PORT); + assert.equal(options.extraVariables?.RUN_TEST_IDS_PORT, '54321'); + assert.equal(options.cwd, uri.fsPath); + assert.equal(options.throwOnStdErr, true); + return true; + }), + ), + typeMoq.Times.once(), + ); + }); + test('Debug launched correctly for pytest', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); + debugLauncher.verify( + (x) => + x.launchDebugger( + typeMoq.It.is((launchOptions) => { + assert.equal(launchOptions.cwd, uri.fsPath); + assert.deepEqual(launchOptions.args, ['--rootdir', myTestPath, '--capture', 'no']); + assert.equal(launchOptions.testProvider, 'pytest'); + assert.equal(launchOptions.pytestPort, '12345'); + assert.equal(launchOptions.pytestUUID, 'uuid123'); + assert.strictEqual(launchOptions.runTestIdsPort, '54321'); + return true; + }), + typeMoq.It.isAny(), + ), + typeMoq.Times.once(), + ); + }); +}); diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index e5495629bf28..7db287a3355c 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -1,118 +1,64 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. - -// import * as assert from 'assert'; -// import * as path from 'path'; -// import * as typemoq from 'typemoq'; -// import { Uri } from 'vscode'; -// import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; -// import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; -// import { ITestServer, TestCommandOptions } from '../../../../client/testing/testController/common/types'; -// import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; - -// suite('Unittest test execution adapter', () => { -// let stubConfigSettings: IConfigurationService; -// let outputChannel: typemoq.IMock; - -// setup(() => { -// stubConfigSettings = ({ -// getSettings: () => ({ -// testing: { unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, -// }), -// } as unknown) as IConfigurationService; -// outputChannel = typemoq.Mock.ofType(); -// }); - -// test('runTests should send the run command to the test server', async () => { -// let options: TestCommandOptions | undefined; - -// const stubTestServer = ({ -// sendCommand(opt: TestCommandOptions, runTestIdPort?: string): Promise { -// delete opt.outChannel; -// options = opt; -// assert(runTestIdPort !== undefined); -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => '123456789', -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); -// const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); -// adapter.runTests(uri, [], false).then(() => { -// const expectedOptions: TestCommandOptions = { -// workspaceFolder: uri, -// command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, -// cwd: uri.fsPath, -// uuid: '123456789', -// debugBool: false, -// testIds: [], -// }; -// assert.deepStrictEqual(options, expectedOptions); -// }); -// }); -// test("onDataReceivedHandler should parse the data if the cwd from the payload matches the test adapter's cwd", async () => { -// const stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => '123456789', -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); -// const data = { status: 'success' }; -// const uuid = '123456789'; - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - -// // triggers runTests flow which will run onDataReceivedHandler and the -// // promise resolves into the parsed data. -// const promise = adapter.runTests(uri, [], false); - -// adapter.onDataReceivedHandler({ uuid, data: JSON.stringify(data) }); - -// const result = await promise; - -// assert.deepStrictEqual(result, data); -// }); -// test("onDataReceivedHandler should ignore the data if the cwd from the payload does not match the test adapter's cwd", async () => { -// const correctUuid = '123456789'; -// const incorrectUuid = '987654321'; -// const stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// createUUID: () => correctUuid, -// } as unknown) as ITestServer; - -// const uri = Uri.file('/foo/bar'); - -// const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - -// // triggers runTests flow which will run onDataReceivedHandler and the -// // promise resolves into the parsed data. -// const promise = adapter.runTests(uri, [], false); - -// const data = { status: 'success' }; -// // will not resolve due to incorrect UUID -// adapter.onDataReceivedHandler({ uuid: incorrectUuid, data: JSON.stringify(data) }); - -// const nextData = { status: 'error' }; -// // will resolve and nextData will be returned as result -// adapter.onDataReceivedHandler({ uuid: correctUuid, data: JSON.stringify(nextData) }); - -// const result = await promise; - -// assert.deepStrictEqual(result, nextData); -// }); -// }); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as path from 'path'; +import * as typemoq from 'typemoq'; +import { Uri } from 'vscode'; +import * as sinon from 'sinon'; +import { IConfigurationService, ITestOutputChannel } from '../../../../client/common/types'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; +import { ITestServer, TestCommandOptions } from '../../../../client/testing/testController/common/types'; +import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; +import * as util from '../../../../client/testing/testController/common/utils'; + +suite('Unittest test execution adapter', () => { + let stubConfigSettings: IConfigurationService; + let outputChannel: typemoq.IMock; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, + }), + } as unknown) as IConfigurationService; + outputChannel = typemoq.Mock.ofType(); + sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + }); + teardown(() => { + sinon.restore(); + }); + + test('runTests should send the run command to the test server', async () => { + let options: TestCommandOptions | undefined; + + const stubTestServer = ({ + sendCommand(opt: TestCommandOptions, runTestIdPort?: string): Promise { + delete opt.outChannel; + options = opt; + assert(runTestIdPort !== undefined); + return Promise.resolve(); + }, + onRunDataReceived: () => { + // no body + }, + createUUID: () => '123456789', + } as unknown) as ITestServer; + + const uri = Uri.file('/foo/bar'); + const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); + + const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); + adapter.runTests(uri, [], false).then(() => { + const expectedOptions: TestCommandOptions = { + workspaceFolder: uri, + command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, + cwd: uri.fsPath, + uuid: '123456789', + debugBool: false, + testIds: [], + }; + assert.deepStrictEqual(options, expectedOptions); + }); + }); +}); From c6e177d2de94e8da6a106baec337a5f5e92b30f7 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH 10/34] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../testController/common/resultResolver.ts | 98 +- .../testing/testController/common/utils.ts | 70 ++ .../testController/workspaceTestAdapter.ts | 13 +- src/test/mocks/vsc/index.ts | 112 +++ .../pytestExecutionAdapter.unit.test.ts | 23 +- .../resultResolver.unit.test.ts | 368 ++++++++ .../testExecutionAdapter.unit.test.ts | 5 +- .../workspaceTestAdapter.unit.test.ts | 883 ++++++++++++------ src/test/vscode-mock.ts | 2 + 9 files changed, 1200 insertions(+), 374 deletions(-) create mode 100644 src/test/testing/testController/resultResolver.unit.test.ts diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 8f8cb859b9d5..49243390ad0f 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -1,41 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { - CancellationToken, - Position, - TestController, - TestItem, - Uri, - Range, - TestMessage, - Location, - TestRun, -} from 'vscode'; +import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode'; import * as util from 'util'; -import * as path from 'path'; -import { - DiscoveredTestItem, - DiscoveredTestNode, - DiscoveredTestPayload, - ExecutionTestPayload, - ITestResultResolver, -} from './types'; +import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; import { TestProvider } from '../../types'; import { traceError, traceLog } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; -import { - DebugTestTag, - ErrorTestItemOptions, - RunTestTag, - clearAllChildren, - createErrorTestItem, - getTestCaseNodes, -} from './testItemUtilities'; +import { clearAllChildren, createErrorTestItem, getTestCaseNodes } from './testItemUtilities'; import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { splitLines } from '../../../common/stringUtils'; -import { fixLogLines } from './utils'; +import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils'; export class PythonResultResolver implements ITestResultResolver { testController: TestController; @@ -253,69 +229,5 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } } -// had to switch the order of the original parameter since required param cannot follow optional. -function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - resultResolver: ITestResultResolver, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - resultResolver.runIdToTestItem.set(child.runID, testItem); - resultResolver.runIdToVSid.set(child.runID, child.id_); - resultResolver.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, resultResolver, token); - } - } - }); -} - -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} - -export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} +// had to switch the order of the original parameter since required param cannot follow optional. diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 232aaccd5341..e92db22747c6 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,11 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as net from 'net'; +import * as path from 'path'; +import { CancellationToken, Position, TestController, TestItem, Uri, Range } from 'vscode'; import { traceError, traceLog, traceVerbose } from '../../../logging'; import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; +import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; +import { DiscoveredTestItem, DiscoveredTestNode, ITestResultResolver } from './types'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); @@ -111,3 +115,69 @@ export async function startTestIdServer(testIds: string[]): Promise { }); return 0; } + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} + +export function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5c7ee9cfe520..f3ea0b9f6193 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import * as path from 'path'; import * as util from 'util'; import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; @@ -10,10 +9,11 @@ import { traceError } from '../../logging'; import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { TestProvider } from '../types'; -import { createErrorTestItem, ErrorTestItemOptions, getTestCaseNodes } from './common/testItemUtilities'; +import { createErrorTestItem, getTestCaseNodes } from './common/testItemUtilities'; import { ITestDiscoveryAdapter, ITestExecutionAdapter, ITestResultResolver } from './common/types'; import { IPythonExecutionFactory } from '../../common/process/types'; import { ITestDebugLauncher } from '../common/types'; +import { buildErrorNodeOptions } from './common/utils'; /** * This class exposes a test-provider-agnostic way of discovering tests. @@ -162,12 +162,3 @@ export class WorkspaceTestAdapter { return Promise.resolve(); } } - -function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} diff --git a/src/test/mocks/vsc/index.ts b/src/test/mocks/vsc/index.ts index 89f4ab1a2d07..092fc67da6c6 100644 --- a/src/test/mocks/vsc/index.ts +++ b/src/test/mocks/vsc/index.ts @@ -6,6 +6,7 @@ import { EventEmitter as NodeEventEmitter } from 'events'; import * as vscode from 'vscode'; + // export * from './range'; // export * from './position'; // export * from './selection'; @@ -443,3 +444,114 @@ export enum LogLevel { */ Error = 5, } + +export class TestMessage { + /** + * Human-readable message text to display. + */ + message: string | MarkdownString; + + /** + * Expected test output. If given with {@link TestMessage.actualOutput actualOutput }, a diff view will be shown. + */ + expectedOutput?: string; + + /** + * Actual test output. If given with {@link TestMessage.expectedOutput expectedOutput }, a diff view will be shown. + */ + actualOutput?: string; + + /** + * Associated file location. + */ + location?: vscode.Location; + + /** + * Creates a new TestMessage that will present as a diff in the editor. + * @param message Message to display to the user. + * @param expected Expected output. + * @param actual Actual output. + */ + static diff(message: string | MarkdownString, expected: string, actual: string): TestMessage { + const testMessage = new TestMessage(message); + testMessage.expectedOutput = expected; + testMessage.actualOutput = actual; + return testMessage; + } + + /** + * Creates a new TestMessage instance. + * @param message The message to show to the user. + */ + constructor(message: string | MarkdownString) { + this.message = message; + } +} + +export interface TestItemCollection extends Iterable<[string, vscode.TestItem]> { + /** + * Gets the number of items in the collection. + */ + readonly size: number; + + /** + * Replaces the items stored by the collection. + * @param items Items to store. + */ + replace(items: readonly vscode.TestItem[]): void; + + /** + * Iterate over each entry in this collection. + * + * @param callback Function to execute for each entry. + * @param thisArg The `this` context used when invoking the handler function. + */ + forEach(callback: (item: vscode.TestItem, collection: TestItemCollection) => unknown, thisArg?: unknown): void; + + /** + * Adds the test item to the children. If an item with the same ID already + * exists, it'll be replaced. + * @param item Item to add. + */ + add(item: vscode.TestItem): void; + + /** + * Removes a single test item from the collection. + * @param itemId Item ID to delete. + */ + delete(itemId: string): void; + + /** + * Efficiently gets a test item by ID, if it exists, in the children. + * @param itemId Item ID to get. + * @returns The found item or undefined if it does not exist. + */ + get(itemId: string): vscode.TestItem | undefined; +} + +/** + * Represents a location inside a resource, such as a line + * inside a text file. + */ +export class Location { + /** + * The resource identifier of this location. + */ + uri: vscode.Uri; + + /** + * The document range of this location. + */ + range: vscode.Range; + + /** + * Creates a new location object. + * + * @param uri The resource identifier. + * @param rangeOrPosition The range or position. Positions will be converted to an empty range. + */ + constructor(uri: vscode.Uri, rangeOrPosition: vscode.Range) { + this.uri = uri; + this.range = rangeOrPosition; + } +} diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index f60429d38e5b..9f4c41ba8309 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -29,6 +29,7 @@ suite('pytest test execution adapter', () => { let debugLauncher: typeMoq.IMock; (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; let myTestPath: string; + let startTestIdServerStub: sinon.SinonStub; setup(() => { testServer = typeMoq.Mock.ofType(); @@ -65,7 +66,7 @@ suite('pytest test execution adapter', () => { deferred.resolve(); return Promise.resolve(); }); - sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + startTestIdServerStub = sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); @@ -75,6 +76,26 @@ suite('pytest test execution adapter', () => { teardown(() => { sinon.restore(); }); + test('startTestIdServer called with correct testIds', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + + const testIds = ['test1id', 'test2id']; + await adapter.runTests(uri, testIds, false, testRun.object, execFactory.object); + + sinon.assert.calledWithExactly(startTestIdServerStub, testIds); + }); test('pytest execution called with correct args', async () => { const uri = Uri.file(myTestPath); const uuid = 'uuid123'; diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts new file mode 100644 index 000000000000..57b321c2e36c --- /dev/null +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -0,0 +1,368 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { TestController, Uri, TestItem, CancellationToken, TestRun, TestItemCollection, Range } from 'vscode'; +import * as typemoq from 'typemoq'; +import * as sinon from 'sinon'; +import { TestProvider } from '../../../client/testing/types'; +import { + DiscoveredTestNode, + DiscoveredTestPayload, + ExecutionTestPayload, +} from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; +import * as util from '../../../client/testing/testController/common/utils'; + +suite('Result Resolver tests', () => { + suite('Test discovery', () => { + let resultResolver: ResultResolver.PythonResultResolver; + let testController: TestController; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let defaultErrorMessage: string; + let blankTestItem: TestItem; + let cancelationToken: CancellationToken; + + setup(() => { + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + + dispose: () => { + // empty + }, + } as unknown) as TestController; + defaultErrorMessage = 'pytest test discovery error (see Output > Python)'; + blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + }); + teardown(() => { + sinon.restore(); + }); + + test('resolveDiscovery calls populate test tree correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const tests: DiscoveredTestNode = { + path: 'path', + name: 'name', + type_: 'folder', + id_: 'id', + children: [], + }; + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + tests, + }; + + // stub out functionality of populateTestTreeStub which is called in resolveDiscovery + const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); + + // call resolve discovery + resultResolver.resolveDiscovery(payload, cancelationToken); + + // assert the stub functions were called with the correct parameters + + // header of populateTestTree is (testController: TestController, testTreeData: DiscoveredTestNode, testRoot: TestItem | undefined, resultResolver: ITestResultResolver, token?: CancellationToken) + sinon.assert.calledWithMatch( + populateTestTreeStub, + testController, // testController + tests, // testTreeData + undefined, // testRoot + resultResolver, // resultResolver + cancelationToken, // token + ); + }); + // what about if the error node already exists: this.testController.items.get(`DiscoveryError:${workspacePath}`); + test('resolveDiscovery should create error node on error with correct params', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const errorMessage = 'error msg A'; + const expectedErrorMessage = `${defaultErrorMessage}\r\n ${errorMessage}`; + + // stub out return values of functions called in resolveDiscovery + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + errors: [errorMessage], + }; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + + // stub out functionality of buildErrorNodeOptions and createErrorTestItem which are called in resolveDiscovery + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + + // call resolve discovery + resultResolver.resolveDiscovery(payload); + + // assert the stub functions were called with the correct parameters + + // header of buildErrorNodeOptions is (uri: Uri, message: string, testType: string) + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, workspaceUri, expectedErrorMessage, testProvider); + // header of createErrorTestItem is (options: ErrorTestItemOptions, testController: TestController, uri: Uri) + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + }); + }); + suite('Test execution result resolver', () => { + let resultResolver: ResultResolver.PythonResultResolver; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let cancelationToken: CancellationToken; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let mockTestItem1: TestItem; + let mockTestItem2: TestItem; + + setup(() => { + // create mock test items + mockTestItem1 = createMockTestItem('mockTestItem1'); + mockTestItem2 = createMockTestItem('mockTestItem2'); + + // create mock testItems to pass into a iterable + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + // create mock testItemCollection + const testItemCollectionMock = typemoq.Mock.ofType(); + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + + // create mock testController + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + + // define functions within runInstance + runInstance = typemoq.Mock.ofType(); + runInstance.setup((r) => r.name).returns(() => 'name'); + runInstance.setup((r) => r.token).returns(() => cancelationToken); + runInstance.setup((r) => r.isPersisted).returns(() => true); + runInstance + .setup((r) => r.enqueued(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('enqueue'); + return undefined; + }); + runInstance + .setup((r) => r.started(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('start'); + }); + + // mock getTestCaseNodes to just return the given testNode added + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => [testNode]); + }); + teardown(() => { + sinon.restore(); + }); + test('resolveExecution handles failed tests correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'failure', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles skipped correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'skipped', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles success correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'success', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles error correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + + const errorPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + error: 'error', + }; + + resultResolver.resolveExecution(errorPayload, runInstance.object); + + // verify that none of these functions are called + + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.never()); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = new Range(0, 0, 0, 0); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 7db287a3355c..88126225a177 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -49,14 +49,15 @@ suite('Unittest test execution adapter', () => { const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - adapter.runTests(uri, [], false).then(() => { + const testIds = ['test1id', 'test2id']; + adapter.runTests(uri, testIds, false).then(() => { const expectedOptions: TestCommandOptions = { workspaceFolder: uri, command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, cwd: uri.fsPath, uuid: '123456789', debugBool: false, - testIds: [], + testIds, }; assert.deepStrictEqual(options, expectedOptions); }); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 42e38d200546..5a2e48130746 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,267 +1,616 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. - -// import * as assert from 'assert'; -// import * as sinon from 'sinon'; -// import * as typemoq from 'typemoq'; - -// import { TestController, TestItem, Uri } from 'vscode'; -// import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; -// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 -// import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; -// import * as Telemetry from '../../../client/telemetry'; -// import { EventName } from '../../../client/telemetry/constants'; -// import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; - -// suite('Workspace test adapter', () => { -// suite('Test discovery', () => { -// let stubTestServer: ITestServer; -// let stubConfigSettings: IConfigurationService; -// let stubResultResolver: ITestResultResolver; - -// let discoverTestsStub: sinon.SinonStub; -// let sendTelemetryStub: sinon.SinonStub; -// let outputChannel: typemoq.IMock; - -// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - -// // Stubbed test controller (see comment around L.40) -// let testController: TestController; -// let log: string[] = []; - -// const sandbox = sinon.createSandbox(); - -// setup(() => { -// stubConfigSettings = ({ -// getSettings: () => ({ -// testing: { unittestArgs: ['--foo'] }, -// }), -// } as unknown) as IConfigurationService; - -// stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// } as unknown) as ITestServer; - -// stubResultResolver = ({ -// resolveDiscovery: () => { -// // no body -// }, -// resolveExecution: () => { -// // no body -// }, -// vsIdToRunId: { -// get: sinon.stub().returns('expectedRunId'), -// }, -// } as unknown) as ITestResultResolver; - -// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); -// // const expectedRunId = 'expectedRunId'; -// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - -// // For some reason the 'tests' namespace in vscode returns undefined. -// // While I figure out how to expose to the tests, they will run -// // against a stub test controller and stub test items. -// const testItem = ({ -// canResolveChildren: false, -// tags: [], -// children: { -// add: () => { -// // empty -// }, -// }, -// } as unknown) as TestItem; - -// testController = ({ -// items: { -// get: () => { -// log.push('get'); -// }, -// add: () => { -// log.push('add'); -// }, -// replace: () => { -// log.push('replace'); -// }, -// delete: () => { -// log.push('delete'); -// }, -// }, -// createTestItem: () => { -// log.push('createTestItem'); -// return testItem; -// }, -// dispose: () => { -// // empty -// }, -// } as unknown) as TestController; - -// // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - -// const mockSendTelemetryEvent = ( -// eventName: EventName, -// _: number | Record | undefined, -// properties: unknown, -// ) => { -// telemetryEvent.push({ -// eventName, -// properties: properties as Record, -// }); -// }; - -// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); -// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); -// outputChannel = typemoq.Mock.ofType(); -// }); - -// teardown(() => { -// telemetryEvent = []; -// log = []; -// testController.dispose(); -// sandbox.restore(); -// }); - -// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { -// discoverTestsStub.resolves(); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { -// discoverTestsStub.callsFake( -// async () => -// new Promise((resolve) => { -// setTimeout(() => { -// // Simulate time taken by discovery. -// resolve(); -// }, 2000); -// }), -// ); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// // Try running discovery twice -// const one = workspaceTestAdapter.discoverTests(testController); -// const two = workspaceTestAdapter.discoverTests(testController); - -// Promise.all([one, two]); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { -// discoverTestsStub.resolves({ status: 'success' }); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.strictEqual(lastEvent.properties.failed, false); -// }); - -// test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { -// discoverTestsStub.rejects(new Error('foo')); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.ok(lastEvent.properties.failed); - -// assert.deepStrictEqual(log, ['createTestItem', 'add']); -// }); - -// /** -// * TODO To test: -// * - successful discovery but no data: delete everything from the test controller -// * - successful discovery with error status: add error node to tree -// * - single root: populate tree if there's no root node -// * - single root: update tree if there's a root node -// * - single root: delete tree if there are no tests in the test data -// * - multiroot: update the correct folders -// */ -// }); -// }); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; + +import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 +import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +import * as Telemetry from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as util from '../../../client/testing/testController/common/utils'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; + +suite('Workspace test adapter', () => { + suite('Test discovery', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + } as unknown) as ITestResultResolver; + + // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); + // const expectedRunId = 'expectedRunId'; + // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); + sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sinon.restore(); + }); + + test('If discovery failed correctly create error node', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + const abc = await workspaceTestAdapter.discoverTests(testController); + console.log(abc); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { + discoverTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); + + Promise.all([one, two]); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.strictEqual(lastEvent.properties.failed, false); + }); + + test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.ok(lastEvent.properties.failed); + }); + }); + suite('Test execution workspace test adapter', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + let executionTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + let resultResolver: ResultResolver.PythonResultResolver; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + const sandbox = sinon.createSandbox(); + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + vsIdToRunId: { + get: sinon.stub().returns('expectedRunId'), + }, + } as unknown) as ITestResultResolver; + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + executionTestsStub = sandbox.stub(UnittestTestExecutionAdapter.prototype, 'runTests'); + sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + runInstance = typemoq.Mock.ofType(); + + const testProvider = 'pytest'; + const workspaceUri = Uri.file('foo'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sandbox.restore(); + }); + test('When executing tests, the right tests should be sent to be executed', async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + resultResolver, + ); + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => + // Custom implementation logic here based on the provided testNode and collection + + // Example implementation: returning a predefined array of TestItem objects + [testNode], + ); + + const mockTestItem1 = createMockTestItem('mockTestItem1'); + const mockTestItem2 = createMockTestItem('mockTestItem2'); + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + // Add as many mock TestItems as needed + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + const testItemCollectionMock = typemoq.Mock.ofType(); + + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, [mockTestItem1, mockTestItem2]); + + runInstance.verify((r) => r.started(typemoq.It.isAny()), typemoq.Times.exactly(2)); + }); + + test("When executing tests, the workspace test adapter should call the test execute adapter's executionTest method", async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution is already running, do not call executionAdapter.runTests again', async () => { + executionTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + Promise.all([one, two]); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution failed correctly create error node', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_RUN_ALL_FAILED); + assert.strictEqual(telemetryEvent.length, 1); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = typemoq.Mock.ofType(); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/vscode-mock.ts b/src/test/vscode-mock.ts index ebbe7ca59e72..44518e7575a7 100644 --- a/src/test/vscode-mock.ts +++ b/src/test/vscode-mock.ts @@ -70,6 +70,8 @@ mockedVSCode.Hover = vscodeMocks.Hover; mockedVSCode.Disposable = vscodeMocks.Disposable as any; mockedVSCode.ExtensionKind = vscodeMocks.ExtensionKind; mockedVSCode.CodeAction = vscodeMocks.CodeAction; +mockedVSCode.TestMessage = vscodeMocks.TestMessage; +mockedVSCode.Location = vscodeMocks.Location; mockedVSCode.EventEmitter = vscodeMocks.EventEmitter; mockedVSCode.CancellationTokenSource = vscodeMocks.CancellationTokenSource; mockedVSCode.CompletionItemKind = vscodeMocks.CompletionItemKind; From 6bdbfb52aeaa4db78b4073f79dac57410500598e Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 20 Jun 2023 11:22:33 -0700 Subject: [PATCH 11/34] fix issue regarding port return and await --- .../testing/testController/common/utils.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index e92db22747c6..f98550d3e72b 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -104,16 +104,23 @@ export async function startTestIdServer(testIds: string[]): Promise { }); // Start the server and wait until it is listening - await startServer() - .then((assignedPort) => { - traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); - return assignedPort; - // if (spawnOptions.extraVariables) spawnOptions.extraVariables.RUN_TEST_IDS_PORT = pytestRunTestIdsPort; - }) - .catch((error) => { - traceError('Error starting server for pytest test ids server:', error); - }); - return 0; + let returnPort = 0; + try { + await startServer() + .then((assignedPort) => { + traceVerbose(`Server started for pytest test ids server and listening on port ${assignedPort}`); + returnPort = assignedPort; + }) + .catch((error) => { + traceError('Error starting server for pytest test ids server:', error); + return 0; + }) + .finally(() => returnPort); + return returnPort; + } catch { + traceError('Error starting server for pytest test ids server, cannot get port.'); + return returnPort; + } } export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { From 4a1bcf9522c776a4a9d8bc18e794bc87e606528a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 20 Jun 2023 14:08:38 -0700 Subject: [PATCH 12/34] remove unneeded prints --- pythonFiles/unittestadapter/execution.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 4695064396cc..17c125e5843a 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -242,7 +242,6 @@ def run_tests( try: client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client_socket.connect(("localhost", run_test_ids_port_int)) - print(f"CLIENT: Server listening on port {run_test_ids_port_int}...") buffer = b"" while True: @@ -263,7 +262,6 @@ def run_tests( buffer = b"" # Process the JSON data - print(f"Received JSON data: {test_ids_from_buffer}") break except json.JSONDecodeError: # JSON decoding error, the complete JSON object is not yet received From a34b637d3475b146efbb1532c8ac79f2790662f6 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 09:04:13 -0700 Subject: [PATCH 13/34] python server sending dynamic test payloads --- pythonFiles/tests/pytestadapter/helpers.py | 69 ++++++++----- .../tests/pytestadapter/test_discovery.py | 25 +++-- .../tests/pytestadapter/test_execution.py | 42 ++++---- .../tests/unittestadapter/test_execution.py | 6 +- pythonFiles/unittestadapter/execution.py | 99 ++++++------------- pythonFiles/vscode_pytest/__init__.py | 44 ++++++--- 6 files changed, 141 insertions(+), 144 deletions(-) diff --git a/pythonFiles/tests/pytestadapter/helpers.py b/pythonFiles/tests/pytestadapter/helpers.py index 013e4bb31fca..c3e01d52170a 100644 --- a/pythonFiles/tests/pytestadapter/helpers.py +++ b/pythonFiles/tests/pytestadapter/helpers.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + import io import json import os @@ -9,7 +10,7 @@ import sys import threading import uuid -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" from typing_extensions import TypedDict @@ -72,21 +73,34 @@ def process_rpc_message(data: str) -> Tuple[Dict[str, Any], str]: str_stream: io.StringIO = io.StringIO(data) length: int = 0 + while True: line: str = str_stream.readline() if CONTENT_LENGTH.lower() in line.lower(): length = int(line[len(CONTENT_LENGTH) :]) break + if not line or line.isspace(): raise ValueError("Header does not contain Content-Length") + while True: line: str = str_stream.readline() if not line or line.isspace(): break raw_json: str = str_stream.read(length) - dict_json: Dict[str, Any] = json.loads(raw_json) - return dict_json, str_stream.read() + return json.loads(raw_json), str_stream.read() + + +def process_rpc_json(data: str) -> List[Dict[str, Any]]: + """Process the JSON data which comes from the server which runs the pytest discovery.""" + json_messages = [] + remaining = data + while remaining: + json_data, remaining = process_rpc_message(remaining) + json_messages.append(json_data) + + return json_messages def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]: @@ -110,53 +124,58 @@ def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]: "PYTHONPATH": os.fspath(pathlib.Path(__file__).parent.parent.parent), } ) + completed = threading.Event() - result: list = [] + result = [] t1: threading.Thread = threading.Thread( - target=_listen_on_socket, args=(listener, result) + target=_listen_on_socket, args=(listener, result, completed) ) t1.start() t2 = threading.Thread( - target=lambda proc_args, proc_env, proc_cwd: subprocess.run( - proc_args, env=proc_env, cwd=proc_cwd - ), - args=(process_args, env, TEST_DATA_PATH), + target=_run_test_code, + args=(process_args, env, TEST_DATA_PATH, completed), ) t2.start() t1.join() t2.join() - a = process_rpc_json(result[0]) - return a if result else None + return process_rpc_json(result[0]) if result else None -def process_rpc_json(data: str) -> List[Dict[str, Any]]: - """Process the JSON data which comes from the server which runs the pytest discovery.""" - json_messages = [] - remaining = data - while remaining: - json_data, remaining = process_rpc_message(remaining) - json_messages.append(json_data) - - return json_messages - - -def _listen_on_socket(listener: socket.socket, result: List[str]): +def _listen_on_socket( + listener: socket.socket, result: List[str], completed: threading.Event +): """Listen on the socket for the JSON data from the server. - Created as a seperate function for clarity in threading. + Created as a separate function for clarity in threading. """ sock, (other_host, other_port) = listener.accept() + listener.settimeout(1) all_data: list = [] while True: data: bytes = sock.recv(1024 * 1024) if not data: - break + if completed.is_set(): + break + else: + try: + sock, (other_host, other_port) = listener.accept() + except socket.timeout: + result.append("".join(all_data)) + return all_data.append(data.decode("utf-8")) result.append("".join(all_data)) +def _run_test_code( + proc_args: List[str], proc_env, proc_cwd: str, completed: threading.Event +): + result = subprocess.run(proc_args, env=proc_env, cwd=proc_cwd) + completed.set() + return result + + def find_test_line_number(test_name: str, test_file_path) -> str: """Function which finds the correct line number for a test by looking for the "test_marker--[test_name]" string. diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 8c0f8dbae68c..c53fc5a0f46f 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import os import shutil -from typing import Any, Dict, List, Optional import pytest @@ -29,11 +28,10 @@ def test_syntax_error(tmp_path): temp_dir.mkdir() p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) - actual_list: Optional[List[Dict[str, Any]]] = runner( - ["--collect-only", os.fspath(p)] - ) - assert actual_list - for actual in actual_list: + actual = runner(["--collect-only", os.fspath(p)]) + if actual: + actual = actual[0] + assert actual assert all(item in actual for item in ("status", "cwd", "error")) assert actual["status"] == "error" assert actual["cwd"] == os.fspath(TEST_DATA_PATH) @@ -46,11 +44,9 @@ def test_parameterized_error_collect(): The json should still be returned but the errors list should be present. """ file_path_str = "error_parametrize_discovery.py" - actual_list: Optional[List[Dict[str, Any]]] = runner( - ["--collect-only", file_path_str] - ) - assert actual_list - for actual in actual_list: + actual = runner(["--collect-only", file_path_str]) + if actual: + actual = actual[0] assert all(item in actual for item in ("status", "cwd", "error")) assert actual["status"] == "error" assert actual["cwd"] == os.fspath(TEST_DATA_PATH) @@ -105,14 +101,15 @@ def test_pytest_collect(file, expected_const): file -- a string with the file or folder to run pytest discovery on. expected_const -- the expected output from running pytest discovery on the file. """ - actual_list: Optional[List[Dict[str, Any]]] = runner( + actual = runner( [ "--collect-only", os.fspath(TEST_DATA_PATH / file), ] ) - assert actual_list - for actual in actual_list: + if actual: + actual = actual[0] + assert actual assert all(item in actual for item in ("status", "cwd", "tests")) assert actual["status"] == "success" assert actual["cwd"] == os.fspath(TEST_DATA_PATH) diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 5b2cbddb1644..d11766027a91 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import os import shutil -from typing import Any, Dict, List, Optional import pytest from tests.pytestadapter import expected_execution_test_output @@ -29,11 +28,10 @@ def test_syntax_error_execution(tmp_path): temp_dir.mkdir() p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) - actual_list: Optional[List[Dict[str, Any]]] = runner( - ["error_syntax_discover.py::test_function"] - ) - assert actual_list - for actual in actual_list: + actual = runner(["error_syntax_discover.py::test_function"]) + if actual: + actual = actual[0] + assert actual assert all(item in actual for item in ("status", "cwd", "error")) assert actual["status"] == "error" assert actual["cwd"] == os.fspath(TEST_DATA_PATH) @@ -45,9 +43,10 @@ def test_bad_id_error_execution(): The json should still be returned but the errors list should be present. """ - actual_list: Optional[List[Dict[str, Any]]] = runner(["not/a/real::test_id"]) - assert actual_list - for actual in actual_list: + actual = runner(["not/a/real::test_id"]) + if actual: + actual = actual[0] + assert actual assert all(item in actual for item in ("status", "cwd", "error")) assert actual["status"] == "error" assert actual["cwd"] == os.fspath(TEST_DATA_PATH) @@ -156,14 +155,17 @@ def test_pytest_execution(test_ids, expected_const): expected_const -- a dictionary of the expected output from running pytest discovery on the files. """ args = test_ids - actual_list: Optional[List[Dict[str, Any]]] = runner(args) - assert actual_list - for actual in actual_list: - assert all(item in actual for item in ("status", "cwd", "result")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - result_data = actual["result"] - for key in result_data: - if result_data[key]["outcome"] == "failure": - result_data[key]["message"] = "ERROR MESSAGE" - assert result_data == expected_const + actual = runner(args) + assert actual + print(actual) + assert len(actual) == len(expected_const) + actual_result_dict = dict() + for a in actual: + assert all(item in a for item in ("status", "cwd", "result")) + assert a["status"] == "success" + assert a["cwd"] == os.fspath(TEST_DATA_PATH) + actual_result_dict.update(a["result"]) + for key in actual_result_dict: + if actual_result_dict[key]["outcome"] == "failure": + actual_result_dict[key]["message"] = "ERROR MESSAGE" + assert actual_result_dict == expected_const diff --git a/pythonFiles/tests/unittestadapter/test_execution.py b/pythonFiles/tests/unittestadapter/test_execution.py index d461ead9ad94..0df56dbf6eff 100644 --- a/pythonFiles/tests/unittestadapter/test_execution.py +++ b/pythonFiles/tests/unittestadapter/test_execution.py @@ -21,11 +21,11 @@ "--uuid", "fake-uuid", ], - (111, "fake-uuid"), + (111, "fake-uuid", None), ), ( ["--port", "111", "--uuid", "fake-uuid"], - (111, "fake-uuid"), + (111, "fake-uuid", None), ), ( [ @@ -36,7 +36,7 @@ "-v", "-s", ], - (111, "fake-uuid"), + (111, "fake-uuid", None), ), ], ) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 17c125e5843a..bd9ee613586b 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -6,20 +6,16 @@ import json import os import pathlib -import socket import sys import traceback import unittest from types import TracebackType from typing import Dict, List, Optional, Tuple, Type, Union -script_dir = pathlib.Path(__file__).parent.parent -sys.path.append(os.fspath(script_dir)) -sys.path.append(os.fspath(script_dir / "lib" / "python")) -from testing_tools import process_json_util - +directory_path = pathlib.Path(__file__).parent.parent / "lib" / "python" # Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + sys.path.insert(0, PYTHON_FILES) # Add the lib path to sys.path to find the typing_extensions module. sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python")) @@ -32,7 +28,7 @@ def parse_execution_cli_args( args: List[str], -) -> Tuple[int, Union[str, None]]: +) -> Tuple[int, Union[str, None], List[str]]: """Parse command-line arguments that should be processed by the script. So far this includes the port number that it needs to connect to, the uuid passed by the TS side, @@ -46,14 +42,18 @@ def parse_execution_cli_args( arg_parser = argparse.ArgumentParser() arg_parser.add_argument("--port", default=DEFAULT_PORT) arg_parser.add_argument("--uuid") + arg_parser.add_argument("--testids", nargs="+") parsed_args, _ = arg_parser.parse_known_args(args) - return (int(parsed_args.port), parsed_args.uuid) + return (int(parsed_args.port), parsed_args.uuid, parsed_args.testids) ErrorType = Union[ Tuple[Type[BaseException], BaseException, TracebackType], Tuple[None, None, None] ] +PORT = 0 +UUID = 0 +START_DIR = "" class TestOutcomeEnum(str, enum.Enum): @@ -150,6 +150,9 @@ def formatResult( } self.formatted[test_id] = result + if PORT == 0 or UUID == 0: + print("Error sending response, port or uuid unknown to python server.") + send_run_data(result, PORT, UUID) class TestExecutionStatus(str, enum.Enum): @@ -225,67 +228,15 @@ def run_tests( return payload -if __name__ == "__main__": - # Get unittest test execution arguments. - argv = sys.argv[1:] - index = argv.index("--udiscovery") - - start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) - - run_test_ids_port = os.environ.get("RUN_TEST_IDS_PORT") - run_test_ids_port_int = ( - int(run_test_ids_port) if run_test_ids_port is not None else 0 - ) - - # get data from socket - test_ids_from_buffer = [] - try: - client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - client_socket.connect(("localhost", run_test_ids_port_int)) - buffer = b"" - - while True: - # Receive the data from the client - data = client_socket.recv(1024 * 1024) - if not data: - break - - # Append the received data to the buffer - buffer += data - - try: - # Try to parse the buffer as JSON - test_ids_from_buffer = process_json_util.process_rpc_json( - buffer.decode("utf-8") - ) - # Clear the buffer as complete JSON object is received - buffer = b"" - - # Process the JSON data - break - except json.JSONDecodeError: - # JSON decoding error, the complete JSON object is not yet received - continue - except socket.error as e: - print(f"Error: Could not connect to runTestIdsPort: {e}") - print("Error: Could not connect to runTestIdsPort") - - port, uuid = parse_execution_cli_args(argv[:index]) - if test_ids_from_buffer: - # Perform test execution. - payload = run_tests( - start_dir, test_ids_from_buffer, pattern, top_level_dir, uuid - ) - else: - cwd = os.path.abspath(start_dir) - status = TestExecutionStatus.error - payload: PayloadDict = { - "cwd": cwd, - "status": status, - "error": "No test ids received from buffer", - } +def send_run_data(raw_data, port, uuid): + # Build the request data (it has to be a POST request or the Node side will not process it), and send it. + status = raw_data["outcome"] + cwd = os.path.abspath(START_DIR) + test_id = raw_data["test"] + test_dict = {} + test_dict[test_id] = raw_data + payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} - # Build the request data and send it. addr = ("localhost", port) data = json.dumps(payload) request = f"""Content-Length: {len(data)} @@ -300,3 +251,15 @@ def run_tests( except Exception as e: print(f"Error sending response: {e}") print(f"Request data: {request}") + + +if __name__ == "__main__": + # Get unittest test execution arguments. + argv = sys.argv[1:] + index = argv.index("--udiscovery") + + START_DIR, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + + # Perform test execution. + PORT, UUID, testids = parse_execution_cli_args(argv[:index]) + payload = run_tests(START_DIR, testids, pattern, top_level_dir, UUID) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index f9f147c381a9..de201529fda1 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -120,7 +120,6 @@ class testRunResultDict(Dict[str, Dict[str, TestOutcome]]): tests: Dict[str, TestOutcome] -collected_tests = testRunResultDict() IS_DISCOVERY = False @@ -130,6 +129,9 @@ def pytest_load_initial_conftests(early_config, parser, args): IS_DISCOVERY = True +collected_tests_so_far = list() + + def pytest_report_teststatus(report, config): """ A pytest hook that is called when a test is called. It is called 3 times per test, @@ -138,6 +140,7 @@ def pytest_report_teststatus(report, config): report -- the report on the test setup, call, and teardown. config -- configuration object. """ + cwd = pathlib.Path.cwd() if report.when == "call": traceback = None @@ -148,13 +151,23 @@ def pytest_report_teststatus(report, config): elif report.failed: report_value = "failure" message = report.longreprtext - item_result = create_test_outcome( - report.nodeid, - report_value, - message, - traceback, - ) - collected_tests[report.nodeid] = item_result + node_id = str(report.nodeid) + if node_id not in collected_tests_so_far: + collected_tests_so_far.append(node_id) + item_result = create_test_outcome( + node_id, + report_value, + message, + traceback, + ) + collected_test = testRunResultDict() + collected_test[node_id] = item_result + execution_post( + os.fsdecode(cwd), + "success", + collected_test if collected_test else None, + ) + # collected_tests[report.nodeid] = item_result ERROR_MESSAGE_CONST = { @@ -196,7 +209,9 @@ def pytest_sessionfinish(session, exitstatus): ) post_response(os.fsdecode(cwd), session_node) except Exception as e: - f"Error Occurred, description: {e.args[0] if e.args and e.args[0] else ''} traceback: {(traceback.format_exc() if e.__traceback__ else '')}" + ERRORS.append( + f"Error Occurred, traceback: {(traceback.format_exc() if e.__traceback__ else '')}" + ) errorNode: TestNode = { "name": "", "path": cwd, @@ -213,11 +228,12 @@ def pytest_sessionfinish(session, exitstatus): f"Pytest exited with error status: {exitstatus}, {ERROR_MESSAGE_CONST[exitstatus]}" ) exitstatus_bool = "error" - execution_post( - os.fsdecode(cwd), - exitstatus_bool, - collected_tests if collected_tests else None, - ) + + execution_post( + os.fsdecode(cwd), + exitstatus_bool, + None, + ) def build_test_tree(session: pytest.Session) -> TestNode: From b84fb743c2e9956c1069339fcc56b6c7e3e65758 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 11:15:23 -0700 Subject: [PATCH 14/34] Remote resolver (#21332) This is branch will serve as a feature branch for all changes related to switching to the remote resolver. This will include - switching from using the testAdapter to parse the return data to now using this new class resultResolver - adding tests for all testAdapters, fixing for server and adding for resultResolver - moving sendCommand to a new file, out of the server, and getting pytest to adopt it - moving the server which send the test IDs to a new file and adopt it for both pytest and unittest - write tests for these two new files. --- src/client/testing/testController/common/resultResolver.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 49243390ad0f..db679b03f0ef 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -229,5 +229,3 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } } - -// had to switch the order of the original parameter since required param cannot follow optional. From 932dc39cb85cb6d3ace1ebe25e4d8b9bb71000b8 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH 15/34] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../testing/testController/common/utils.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index f98550d3e72b..80de1e082420 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -188,3 +188,69 @@ export function populateTestTree( function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { return test.type_ === 'test'; } + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} + +export function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} From c81e9ac179a960d1f5ae91b0d9b61bafdc7745bb Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 09:14:00 -0700 Subject: [PATCH 16/34] rebasing --- .../testing/testController/common/utils.ts | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 80de1e082420..f98550d3e72b 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -188,69 +188,3 @@ export function populateTestTree( function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { return test.type_ === 'test'; } - -export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} - -export function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - resultResolver: ITestResultResolver, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - resultResolver.runIdToTestItem.set(child.runID, testItem); - resultResolver.runIdToVSid.set(child.runID, child.id_); - resultResolver.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, resultResolver, token); - } - } - }); -} - -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} From 65cc6e9cf5b7f0bf39ccff212dcfe11d31d56449 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 09:26:43 -0700 Subject: [PATCH 17/34] formatting --- pythonFiles/unittestadapter/execution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 504fef6a5eb6..57103e3f4c59 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -228,7 +228,6 @@ def run_tests( return payload - def send_run_data(raw_data, port, uuid): # Build the request data (it has to be a POST request or the Node side will not process it), and send it. status = raw_data["outcome"] From c240da5606c46c3683f3ae35f3fefe884fe37b3e Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 11:40:39 -0700 Subject: [PATCH 18/34] adding dynamic run results --- pythonFiles/unittestadapter/execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 57103e3f4c59..e5eb86ea2c4b 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -229,6 +229,7 @@ def run_tests( def send_run_data(raw_data, port, uuid): + print("sending run data") # Build the request data (it has to be a POST request or the Node side will not process it), and send it. status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) From e257501d63496808b63ca386637e4176a3066790 Mon Sep 17 00:00:00 2001 From: Ladislau Szomoru <3372902+lszomoru@users.noreply.github.com> Date: Wed, 21 Jun 2023 02:01:18 +0200 Subject: [PATCH 19/34] Engineering - add TSAOptions (#21460) --- build/azure-pipeline.stable.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build/azure-pipeline.stable.yml b/build/azure-pipeline.stable.yml index 50ccbb9fff7a..5c8444de9b2e 100644 --- a/build/azure-pipeline.stable.yml +++ b/build/azure-pipeline.stable.yml @@ -63,7 +63,7 @@ extends: - script: | python ./build/update_ext_version.py --release --for-publishing displayName: Update build number - + - script: | python ./build/update_package_file.py displayName: Update telemetry in package.json @@ -73,3 +73,16 @@ extends: - script: gulp prePublishBundle displayName: Build + tsa: + enabled: true + options: + codebaseName: 'devdiv_$(Build.Repository.Name)' + serviceTreeID: '6e6194bc-7baa-4486-86d0-9f5419626d46' + instanceUrl: 'https://devdiv.visualstudio.com/defaultcollection' + projectName: 'DevDiv' + areaPath: "DevDiv\\VS Code (compliance tracking only)\\Visual Studio Code Python Extensions" + notificationAliases: + - 'stbatt@microsoft.com' + - 'lszomoru@microsoft.com' + - 'brcan@microsoft.com' + - 'kanadig@microsoft.com' From bfdb6acc7f8be10f0eb8563cc9bf22f74c4df199 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Wed, 21 Jun 2023 10:28:34 -0700 Subject: [PATCH 20/34] Result resolver feature branch (#21457) fixes https://github.com/microsoft/vscode-python/issues/21394 --- pythonFiles/unittestadapter/execution.py | 65 +++++++++++++++++++++--- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index e5eb86ea2c4b..6740cc66c66d 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -6,12 +6,15 @@ import json import os import pathlib +import socket import sys import traceback import unittest from types import TracebackType from typing import Dict, List, Optional, Tuple, Type, Union +from pythonFiles.testing_tools import process_json_util + directory_path = pathlib.Path(__file__).parent.parent / "lib" / "python" # Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -28,7 +31,7 @@ def parse_execution_cli_args( args: List[str], -) -> Tuple[int, Union[str, None], List[str]]: +) -> Tuple[int, Union[str, None]]: """Parse command-line arguments that should be processed by the script. So far this includes the port number that it needs to connect to, the uuid passed by the TS side, @@ -42,10 +45,9 @@ def parse_execution_cli_args( arg_parser = argparse.ArgumentParser() arg_parser.add_argument("--port", default=DEFAULT_PORT) arg_parser.add_argument("--uuid") - arg_parser.add_argument("--testids", nargs="+") parsed_args, _ = arg_parser.parse_known_args(args) - return (int(parsed_args.port), parsed_args.uuid, parsed_args.testids) + return (int(parsed_args.port), parsed_args.uuid) ErrorType = Union[ @@ -258,8 +260,57 @@ def send_run_data(raw_data, port, uuid): argv = sys.argv[1:] index = argv.index("--udiscovery") - START_DIR, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + + run_test_ids_port = os.environ.get("RUN_TEST_IDS_PORT") + run_test_ids_port_int = ( + int(run_test_ids_port) if run_test_ids_port is not None else 0 + ) - # Perform test execution. - PORT, UUID, testids = parse_execution_cli_args(argv[:index]) - payload = run_tests(START_DIR, testids, pattern, top_level_dir, UUID) + # get data from socket + test_ids_from_buffer = [] + try: + client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_socket.connect(("localhost", run_test_ids_port_int)) + buffer = b"" + + while True: + # Receive the data from the client + data = client_socket.recv(1024 * 1024) + if not data: + break + + # Append the received data to the buffer + buffer += data + + try: + # Try to parse the buffer as JSON + test_ids_from_buffer = process_json_util.process_rpc_json( + buffer.decode("utf-8") + ) + # Clear the buffer as complete JSON object is received + buffer = b"" + + # Process the JSON data + break + except json.JSONDecodeError: + # JSON decoding error, the complete JSON object is not yet received + continue + except socket.error as e: + print(f"Error: Could not connect to runTestIdsPort: {e}") + print("Error: Could not connect to runTestIdsPort") + + port, uuid = parse_execution_cli_args(argv[:index]) + if test_ids_from_buffer: + # Perform test execution. + payload = run_tests( + start_dir, test_ids_from_buffer, pattern, top_level_dir, uuid + ) + else: + cwd = os.path.abspath(start_dir) + status = TestExecutionStatus.error + payload: PayloadDict = { + "cwd": cwd, + "status": status, + "error": "No test ids received from buffer", + } From 1fa950d1efb08d68f2446a21b26adcaf6a83b976 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 11:15:23 -0700 Subject: [PATCH 21/34] Remote resolver (#21332) This is branch will serve as a feature branch for all changes related to switching to the remote resolver. This will include - switching from using the testAdapter to parse the return data to now using this new class resultResolver - adding tests for all testAdapters, fixing for server and adding for resultResolver - moving sendCommand to a new file, out of the server, and getting pytest to adopt it - moving the server which send the test IDs to a new file and adopt it for both pytest and unittest - write tests for these two new files. --- .../workspaceTestAdapter.unit.test.ts | 281 ++++++++++-------- 1 file changed, 164 insertions(+), 117 deletions(-) diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 5a2e48130746..e58e10e5325c 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,9 +1,9 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. -import * as assert from 'assert'; -import * as sinon from 'sinon'; -import * as typemoq from 'typemoq'; +// import * as assert from 'assert'; +// import * as sinon from 'sinon'; +// import * as typemoq from 'typemoq'; import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; @@ -23,15 +23,15 @@ suite('Workspace test adapter', () => { let stubConfigSettings: IConfigurationService; let stubResultResolver: ITestResultResolver; - let discoverTestsStub: sinon.SinonStub; - let sendTelemetryStub: sinon.SinonStub; - let outputChannel: typemoq.IMock; +// let discoverTestsStub: sinon.SinonStub; +// let sendTelemetryStub: sinon.SinonStub; +// let outputChannel: typemoq.IMock; - let telemetryEvent: { eventName: EventName; properties: Record }[] = []; +// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - // Stubbed test controller (see comment around L.40) - let testController: TestController; - let log: string[] = []; +// // Stubbed test controller (see comment around L.40) +// let testController: TestController; +// let log: string[] = []; setup(() => { stubConfigSettings = ({ @@ -40,14 +40,14 @@ suite('Workspace test adapter', () => { }), } as unknown) as IConfigurationService; - stubTestServer = ({ - sendCommand(): Promise { - return Promise.resolve(); - }, - onDataReceived: () => { - // no body - }, - } as unknown) as ITestServer; +// stubTestServer = ({ +// sendCommand(): Promise { +// return Promise.resolve(); +// }, +// onDataReceived: () => { +// // no body +// }, +// } as unknown) as ITestServer; stubResultResolver = ({ resolveDiscovery: () => { @@ -75,42 +75,46 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestItem; - testController = ({ - items: { - get: () => { - log.push('get'); - }, - add: () => { - log.push('add'); - }, - replace: () => { - log.push('replace'); - }, - delete: () => { - log.push('delete'); - }, - }, - createTestItem: () => { - log.push('createTestItem'); - return testItem; - }, - dispose: () => { - // empty - }, - } as unknown) as TestController; - - // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - - const mockSendTelemetryEvent = ( - eventName: EventName, - _: number | Record | undefined, - properties: unknown, - ) => { - telemetryEvent.push({ - eventName, - properties: properties as Record, - }); - }; +// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); +// // const expectedRunId = 'expectedRunId'; +// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + +// // For some reason the 'tests' namespace in vscode returns undefined. +// // While I figure out how to expose to the tests, they will run +// // against a stub test controller and stub test items. +// const testItem = ({ +// canResolveChildren: false, +// tags: [], +// children: { +// add: () => { +// // empty +// }, +// }, +// } as unknown) as TestItem; + +// testController = ({ +// items: { +// get: () => { +// log.push('get'); +// }, +// add: () => { +// log.push('add'); +// }, +// replace: () => { +// log.push('replace'); +// }, +// delete: () => { +// log.push('delete'); +// }, +// }, +// createTestItem: () => { +// log.push('createTestItem'); +// return testItem; +// }, +// dispose: () => { +// // empty +// }, +// } as unknown) as TestController; discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); @@ -171,8 +175,10 @@ suite('Workspace test adapter', () => { sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); }); - test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { - discoverTestsStub.resolves(); +// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); +// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); +// outputChannel = typemoq.Mock.ofType(); +// }); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -192,21 +198,28 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { - discoverTestsStub.callsFake( - async () => - new Promise((resolve) => { - setTimeout(() => { - // Simulate time taken by discovery. - resolve(); - }, 2000); - }), - ); +// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { +// discoverTestsStub.resolves(); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// await workspaceTestAdapter.discoverTests(testController); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -226,28 +239,43 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - // Try running discovery twice - const one = workspaceTestAdapter.discoverTests(testController); - const two = workspaceTestAdapter.discoverTests(testController); - - Promise.all([one, two]); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { - discoverTestsStub.resolves({ status: 'success' }); - - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); +// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { +// discoverTestsStub.callsFake( +// async () => +// new Promise((resolve) => { +// setTimeout(() => { +// // Simulate time taken by discovery. +// resolve(); +// }, 2000); +// }), +// ); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); + +// // Try running discovery twice +// const one = workspaceTestAdapter.discoverTests(testController); +// const two = workspaceTestAdapter.discoverTests(testController); + +// Promise.all([one, two]); + +// sinon.assert.calledOnce(discoverTestsStub); +// }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -257,28 +285,33 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - await workspaceTestAdapter.discoverTests(testController); +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); - const lastEvent = telemetryEvent[1]; - assert.strictEqual(lastEvent.properties.failed, false); - }); +// await workspaceTestAdapter.discoverTests(testController); - test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { - discoverTestsStub.rejects(new Error('foo')); +// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); +// assert.strictEqual(telemetryEvent.length, 2); - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); +// const lastEvent = telemetryEvent[1]; +// assert.strictEqual(lastEvent.properties.failed, false); +// }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -288,10 +321,24 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); - assert.strictEqual(telemetryEvent.length, 2); +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); + +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); const lastEvent = telemetryEvent[1]; assert.ok(lastEvent.properties.failed); From ebecd18dcffcabf934b7f7efeaca03538e49e36d Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 13 Jun 2023 16:30:37 -0700 Subject: [PATCH 22/34] Execution adapter tests (#21421) adds test coverage for `testExecutionAdapter.ts` and `pytestExecutionAdapter.ts`. --- .../testController/unittest/testExecutionAdapter.unit.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 88126225a177..75b74cebb455 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -51,6 +51,7 @@ suite('Unittest test execution adapter', () => { const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); const testIds = ['test1id', 'test2id']; adapter.runTests(uri, testIds, false).then(() => { + adapter.runTests(uri, [], false).then(() => { const expectedOptions: TestCommandOptions = { workspaceFolder: uri, command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, From 7f9ef9486af1ec557f52041c44047a9bacb036fe Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH 23/34] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../testing/testController/common/utils.ts | 66 ++++ .../workspaceTestAdapter.unit.test.ts | 327 +++++++++--------- 2 files changed, 229 insertions(+), 164 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index f98550d3e72b..80de1e082420 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -188,3 +188,69 @@ export function populateTestTree( function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { return test.type_ === 'test'; } + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} + +export function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index e58e10e5325c..fa64f2cf6687 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,9 +1,9 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. -// import * as assert from 'assert'; -// import * as sinon from 'sinon'; -// import * as typemoq from 'typemoq'; +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; @@ -23,15 +23,15 @@ suite('Workspace test adapter', () => { let stubConfigSettings: IConfigurationService; let stubResultResolver: ITestResultResolver; -// let discoverTestsStub: sinon.SinonStub; -// let sendTelemetryStub: sinon.SinonStub; -// let outputChannel: typemoq.IMock; + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; -// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; -// // Stubbed test controller (see comment around L.40) -// let testController: TestController; -// let log: string[] = []; + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; setup(() => { stubConfigSettings = ({ @@ -40,14 +40,14 @@ suite('Workspace test adapter', () => { }), } as unknown) as IConfigurationService; -// stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// } as unknown) as ITestServer; + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + } as unknown) as ITestResultResolver; stubResultResolver = ({ resolveDiscovery: () => { @@ -75,46 +75,44 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestItem; -// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); -// // const expectedRunId = 'expectedRunId'; -// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - -// // For some reason the 'tests' namespace in vscode returns undefined. -// // While I figure out how to expose to the tests, they will run -// // against a stub test controller and stub test items. -// const testItem = ({ -// canResolveChildren: false, -// tags: [], -// children: { -// add: () => { -// // empty -// }, -// }, -// } as unknown) as TestItem; - -// testController = ({ -// items: { -// get: () => { -// log.push('get'); -// }, -// add: () => { -// log.push('add'); -// }, -// replace: () => { -// log.push('replace'); -// }, -// delete: () => { -// log.push('delete'); -// }, -// }, -// createTestItem: () => { -// log.push('createTestItem'); -// return testItem; -// }, -// dispose: () => { -// // empty -// }, -// } as unknown) as TestController; + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); @@ -175,10 +173,67 @@ suite('Workspace test adapter', () => { sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); }); -// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); -// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); -// outputChannel = typemoq.Mock.ofType(); -// }); + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sinon.restore(); + }); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -198,28 +253,12 @@ suite('Workspace test adapter', () => { stubResultResolver, ); -// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { -// discoverTestsStub.resolves(); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -239,44 +278,29 @@ suite('Workspace test adapter', () => { stubResultResolver, ); -// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { -// discoverTestsStub.callsFake( -// async () => -// new Promise((resolve) => { -// setTimeout(() => { -// // Simulate time taken by discovery. -// resolve(); -// }, 2000); -// }), -// ); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// // Try running discovery twice -// const one = workspaceTestAdapter.discoverTests(testController); -// const two = workspaceTestAdapter.discoverTests(testController); - -// Promise.all([one, two]); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', testDiscoveryAdapter, @@ -285,33 +309,25 @@ suite('Workspace test adapter', () => { stubResultResolver, ); -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); + Promise.all([one, two]); -// await workspaceTestAdapter.discoverTests(testController); + sinon.assert.calledOnce(discoverTestsStub); + }); -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); -// const lastEvent = telemetryEvent[1]; -// assert.strictEqual(lastEvent.properties.failed, false); -// }); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -321,24 +337,7 @@ suite('Workspace test adapter', () => { stubResultResolver, ); -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); + await workspaceTestAdapter.discoverTests(testController); const lastEvent = telemetryEvent[1]; assert.ok(lastEvent.properties.failed); From 8cb9c939fb6a472aa50651bc2a7e047568940158 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Mon, 12 Jun 2023 11:15:23 -0700 Subject: [PATCH 24/34] Remote resolver (#21332) This is branch will serve as a feature branch for all changes related to switching to the remote resolver. This will include - switching from using the testAdapter to parse the return data to now using this new class resultResolver - adding tests for all testAdapters, fixing for server and adding for resultResolver - moving sendCommand to a new file, out of the server, and getting pytest to adopt it - moving the server which send the test IDs to a new file and adopt it for both pytest and unittest - write tests for these two new files. --- .../workspaceTestAdapter.unit.test.ts | 163 +++++++++--------- 1 file changed, 86 insertions(+), 77 deletions(-) diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index fa64f2cf6687..6355a4768734 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,9 +1,9 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. +// // Copyright (c) Microsoft Corporation. All rights reserved. +// // Licensed under the MIT License. -import * as assert from 'assert'; -import * as sinon from 'sinon'; -import * as typemoq from 'typemoq'; +// import * as assert from 'assert'; +// import * as sinon from 'sinon'; +// import * as typemoq from 'typemoq'; import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; @@ -23,15 +23,15 @@ suite('Workspace test adapter', () => { let stubConfigSettings: IConfigurationService; let stubResultResolver: ITestResultResolver; - let discoverTestsStub: sinon.SinonStub; - let sendTelemetryStub: sinon.SinonStub; - let outputChannel: typemoq.IMock; +// let discoverTestsStub: sinon.SinonStub; +// let sendTelemetryStub: sinon.SinonStub; +// let outputChannel: typemoq.IMock; - let telemetryEvent: { eventName: EventName; properties: Record }[] = []; +// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - // Stubbed test controller (see comment around L.40) - let testController: TestController; - let log: string[] = []; +// // Stubbed test controller (see comment around L.40) +// let testController: TestController; +// let log: string[] = []; setup(() => { stubConfigSettings = ({ @@ -75,44 +75,34 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestItem; - // For some reason the 'tests' namespace in vscode returns undefined. - // While I figure out how to expose to the tests, they will run - // against a stub test controller and stub test items. - const testItem = ({ - canResolveChildren: false, - tags: [], - children: { - add: () => { - // empty - }, - }, - } as unknown) as TestItem; - - testController = ({ - items: { - get: () => { - log.push('get'); - }, - add: () => { - log.push('add'); - }, - replace: () => { - log.push('replace'); - }, - delete: () => { - log.push('delete'); - }, - }, - createTestItem: () => { - log.push('createTestItem'); - return testItem; - }, - dispose: () => { - // empty - }, - } as unknown) as TestController; - - // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); +// stubResultResolver = ({ +// resolveDiscovery: () => { +// // no body +// }, +// resolveExecution: () => { +// // no body +// }, +// vsIdToRunId: { +// get: sinon.stub().returns('expectedRunId'), +// }, +// } as unknown) as ITestResultResolver; + +// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); +// // const expectedRunId = 'expectedRunId'; +// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + +// // For some reason the 'tests' namespace in vscode returns undefined. +// // While I figure out how to expose to the tests, they will run +// // against a stub test controller and stub test items. +// const testItem = ({ +// canResolveChildren: false, +// tags: [], +// children: { +// add: () => { +// // empty +// }, +// }, +// } as unknown) as TestItem; discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); @@ -257,8 +247,10 @@ suite('Workspace test adapter', () => { sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); }); - test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { - discoverTestsStub.resolves(); +// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); +// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); +// outputChannel = typemoq.Mock.ofType(); +// }); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -278,10 +270,26 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - await workspaceTestAdapter.discoverTests(testController); - - sinon.assert.calledOnce(discoverTestsStub); - }); +// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { +// discoverTestsStub.resolves(); + +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -562,7 +570,23 @@ suite('Workspace test adapter', () => { const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); - Promise.all([one, two]); +// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const testExecutionAdapter = new UnittestTestExecutionAdapter( +// stubTestServer, +// stubConfigSettings, +// outputChannel.object, +// ); +// const workspaceTestAdapter = new WorkspaceTestAdapter( +// 'unittest', +// testDiscoveryAdapter, +// testExecutionAdapter, +// Uri.parse('foo'), +// stubResultResolver, +// ); sinon.assert.calledOnce(executionTestsStub); }); @@ -570,16 +594,8 @@ suite('Workspace test adapter', () => { test('If execution failed correctly create error node', async () => { executionTestsStub.rejects(new Error('foo')); - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); +// sinon.assert.calledOnce(discoverTestsStub); +// }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -616,16 +632,9 @@ suite('Workspace test adapter', () => { test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { executionTestsStub.rejects(new Error('foo')); - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); +// const lastEvent = telemetryEvent[1]; +// assert.strictEqual(lastEvent.properties.failed, false); +// }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', From c53f99bedaf8aed86775544fdaeefe4df0093cff Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH 25/34] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../workspaceTestAdapter.unit.test.ts | 168 +++++++++--------- 1 file changed, 82 insertions(+), 86 deletions(-) diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 6355a4768734..5962d38782e6 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,9 +1,9 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. -// import * as assert from 'assert'; -// import * as sinon from 'sinon'; -// import * as typemoq from 'typemoq'; +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; @@ -23,15 +23,15 @@ suite('Workspace test adapter', () => { let stubConfigSettings: IConfigurationService; let stubResultResolver: ITestResultResolver; -// let discoverTestsStub: sinon.SinonStub; -// let sendTelemetryStub: sinon.SinonStub; -// let outputChannel: typemoq.IMock; + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; -// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; -// // Stubbed test controller (see comment around L.40) -// let testController: TestController; -// let log: string[] = []; + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; setup(() => { stubConfigSettings = ({ @@ -75,34 +75,46 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestItem; -// stubResultResolver = ({ -// resolveDiscovery: () => { -// // no body -// }, -// resolveExecution: () => { -// // no body -// }, -// vsIdToRunId: { -// get: sinon.stub().returns('expectedRunId'), -// }, -// } as unknown) as ITestResultResolver; - -// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); -// // const expectedRunId = 'expectedRunId'; -// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - -// // For some reason the 'tests' namespace in vscode returns undefined. -// // While I figure out how to expose to the tests, they will run -// // against a stub test controller and stub test items. -// const testItem = ({ -// canResolveChildren: false, -// tags: [], -// children: { -// add: () => { -// // empty -// }, -// }, -// } as unknown) as TestItem; + // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); + // const expectedRunId = 'expectedRunId'; + // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); @@ -247,10 +259,12 @@ suite('Workspace test adapter', () => { sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); }); -// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); -// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); -// outputChannel = typemoq.Mock.ofType(); -// }); + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sinon.restore(); + }); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -270,26 +284,24 @@ suite('Workspace test adapter', () => { stubResultResolver, ); -// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { -// discoverTestsStub.resolves(); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -570,23 +582,8 @@ suite('Workspace test adapter', () => { const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); sinon.assert.calledOnce(executionTestsStub); }); @@ -594,8 +591,8 @@ suite('Workspace test adapter', () => { test('If execution failed correctly create error node', async () => { executionTestsStub.rejects(new Error('foo')); -// sinon.assert.calledOnce(discoverTestsStub); -// }); + sinon.assert.calledOnce(discoverTestsStub); + }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -632,9 +629,8 @@ suite('Workspace test adapter', () => { test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { executionTestsStub.rejects(new Error('foo')); -// const lastEvent = telemetryEvent[1]; -// assert.strictEqual(lastEvent.properties.failed, false); -// }); + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', From 0c65dc61a11bb1daf987eaab81d76ca4c499aa05 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 11:49:30 -0700 Subject: [PATCH 26/34] fixing conflicts --- .../testing/testController/common/utils.ts | 66 ------ .../testExecutionAdapter.unit.test.ts | 1 - .../workspaceTestAdapter.unit.test.ts | 203 +++++++----------- 3 files changed, 76 insertions(+), 194 deletions(-) diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 80de1e082420..f98550d3e72b 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -188,69 +188,3 @@ export function populateTestTree( function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { return test.type_ === 'test'; } - -export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} - -export function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - resultResolver: ITestResultResolver, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - resultResolver.runIdToTestItem.set(child.runID, testItem); - resultResolver.runIdToVSid.set(child.runID, child.id_); - resultResolver.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, resultResolver, token); - } - } - }); -} - -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 75b74cebb455..88126225a177 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -51,7 +51,6 @@ suite('Unittest test execution adapter', () => { const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); const testIds = ['test1id', 'test2id']; adapter.runTests(uri, testIds, false).then(() => { - adapter.runTests(uri, [], false).then(() => { const expectedOptions: TestCommandOptions = { workspaceFolder: uri, command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 5962d38782e6..5a2e48130746 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -40,14 +40,14 @@ suite('Workspace test adapter', () => { }), } as unknown) as IConfigurationService; - stubResultResolver = ({ - resolveDiscovery: () => { - // no body + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); }, - resolveExecution: () => { + onDataReceived: () => { // no body }, - } as unknown) as ITestResultResolver; + } as unknown) as ITestServer; stubResultResolver = ({ resolveDiscovery: () => { @@ -75,23 +75,6 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestItem; - // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); - // const expectedRunId = 'expectedRunId'; - // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - - // For some reason the 'tests' namespace in vscode returns undefined. - // While I figure out how to expose to the tests, they will run - // against a stub test controller and stub test items. - const testItem = ({ - canResolveChildren: false, - tags: [], - children: { - add: () => { - // empty - }, - }, - } as unknown) as TestItem; - testController = ({ items: { get: () => { @@ -116,6 +99,19 @@ suite('Workspace test adapter', () => { }, } as unknown) as TestController; + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); outputChannel = typemoq.Mock.ofType(); @@ -175,12 +171,8 @@ suite('Workspace test adapter', () => { sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); }); - teardown(() => { - telemetryEvent = []; - log = []; - testController.dispose(); - sinon.restore(); - }); + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -200,42 +192,21 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); + await workspaceTestAdapter.discoverTests(testController); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - stubResultResolver, - ); + sinon.assert.calledOnce(discoverTestsStub); + }); - const blankTestItem = ({ - canResolveChildren: false, - tags: [], - children: { - add: () => { - // empty - }, - }, - } as unknown) as TestItem; - const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { - id: 'id', - label: 'label', - error: 'error', - }; - const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); - const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); - const testProvider = 'unittest'; + test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { + discoverTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -255,34 +226,17 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); - sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); - }); + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); - teardown(() => { - telemetryEvent = []; - log = []; - testController.dispose(); - sinon.restore(); + Promise.all([one, two]); + + sinon.assert.calledOnce(discoverTestsStub); }); - const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const testExecutionAdapter = new UnittestTestExecutionAdapter( - stubTestServer, - stubConfigSettings, - outputChannel.object, - ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - stubResultResolver, - ); + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -303,13 +257,17 @@ suite('Workspace test adapter', () => { stubResultResolver, ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - stubResultResolver, - ); + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.strictEqual(lastEvent.properties.failed, false); + }); + + test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + discoverTestsStub.rejects(new Error('foo')); const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( stubTestServer, @@ -321,33 +279,6 @@ suite('Workspace test adapter', () => { stubConfigSettings, outputChannel.object, ); - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - stubResultResolver, - ); - - // Try running discovery twice - const one = workspaceTestAdapter.discoverTests(testController); - const two = workspaceTestAdapter.discoverTests(testController); - - Promise.all([one, two]); - - sinon.assert.calledOnce(discoverTestsStub); - }); - - test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { - discoverTestsStub.resolves({ status: 'success' }); - - const workspaceTestAdapter = new WorkspaceTestAdapter( - 'unittest', - testDiscoveryAdapter, - testExecutionAdapter, - Uri.parse('foo'), - stubResultResolver, - ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -359,6 +290,9 @@ suite('Workspace test adapter', () => { await workspaceTestAdapter.discoverTests(testController); + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + const lastEvent = telemetryEvent[1]; assert.ok(lastEvent.properties.failed); }); @@ -582,8 +516,7 @@ suite('Workspace test adapter', () => { const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); - test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { - discoverTestsStub.resolves(); + Promise.all([one, two]); sinon.assert.calledOnce(executionTestsStub); }); @@ -591,8 +524,16 @@ suite('Workspace test adapter', () => { test('If execution failed correctly create error node', async () => { executionTestsStub.rejects(new Error('foo')); - sinon.assert.calledOnce(discoverTestsStub); - }); + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', @@ -629,8 +570,16 @@ suite('Workspace test adapter', () => { test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { executionTestsStub.rejects(new Error('foo')); - test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { - discoverTestsStub.resolves({ status: 'success' }); + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); const workspaceTestAdapter = new WorkspaceTestAdapter( 'unittest', From a1b58bbb57e67c65e93856f2c393a1bf0efaefb0 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 13:16:08 -0700 Subject: [PATCH 27/34] fix tests to align with server for testids --- pythonFiles/tests/unittestadapter/test_execution.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonFiles/tests/unittestadapter/test_execution.py b/pythonFiles/tests/unittestadapter/test_execution.py index 0df56dbf6eff..d461ead9ad94 100644 --- a/pythonFiles/tests/unittestadapter/test_execution.py +++ b/pythonFiles/tests/unittestadapter/test_execution.py @@ -21,11 +21,11 @@ "--uuid", "fake-uuid", ], - (111, "fake-uuid", None), + (111, "fake-uuid"), ), ( ["--port", "111", "--uuid", "fake-uuid"], - (111, "fake-uuid", None), + (111, "fake-uuid"), ), ( [ @@ -36,7 +36,7 @@ "-v", "-s", ], - (111, "fake-uuid", None), + (111, "fake-uuid"), ), ], ) From 4a0b541be0a1428e08e2e54ec05f8aae37efbc84 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 14:23:54 -0700 Subject: [PATCH 28/34] fix UUID check --- pythonFiles/unittestadapter/execution.py | 10 ++++----- .../testController/common/resultResolver.ts | 13 ++++++------ .../pytest/pytestDiscoveryAdapter.ts | 5 +---- .../pytest/pytestExecutionAdapter.ts | 19 ++++++++++------- .../unittest/testDiscoveryAdapter.ts | 16 +++++++------- .../unittest/testExecutionAdapter.ts | 21 +++++++++++-------- 6 files changed, 45 insertions(+), 39 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 6740cc66c66d..764e6915bff9 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -13,8 +13,6 @@ from types import TracebackType from typing import Dict, List, Optional, Tuple, Type, Union -from pythonFiles.testing_tools import process_json_util - directory_path = pathlib.Path(__file__).parent.parent / "lib" / "python" # Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -22,9 +20,9 @@ sys.path.insert(0, PYTHON_FILES) # Add the lib path to sys.path to find the typing_extensions module. sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python")) -from testing_tools import socket_manager +from testing_tools import process_json_util, socket_manager from typing_extensions import NotRequired, TypeAlias, TypedDict -from unittestadapter.utils import parse_unittest_args +from utils import parse_unittest_args DEFAULT_PORT = "45454" @@ -300,11 +298,11 @@ def send_run_data(raw_data, port, uuid): print(f"Error: Could not connect to runTestIdsPort: {e}") print("Error: Could not connect to runTestIdsPort") - port, uuid = parse_execution_cli_args(argv[:index]) + PORT, UUID = parse_execution_cli_args(argv[:index]) if test_ids_from_buffer: # Perform test execution. payload = run_tests( - start_dir, test_ids_from_buffer, pattern, top_level_dir, uuid + start_dir, test_ids_from_buffer, pattern, top_level_dir, UUID ) else: cwd = os.path.abspath(start_dir) diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 0248209b75e5..c9726cf04f66 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -24,6 +24,8 @@ export class PythonResultResolver implements ITestResultResolver { public vsIdToRunId: Map; + public subTestStats: Map = new Map(); + constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) { this.testController = testController; this.testProvider = testProvider; @@ -88,7 +90,6 @@ export class PythonResultResolver implements ITestResultResolver { const rawTestExecData = payload; if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { // Map which holds the subtest information for each test item. - const subTestStats: Map = new Map(); // iterate through payload and update the UI accordingly. for (const keyTemp of Object.keys(rawTestExecData.result)) { @@ -163,11 +164,11 @@ export class PythonResultResolver implements ITestResultResolver { const data = rawTestExecData.result[keyTemp]; // find the subtest's parent test item if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); + const subtestStats = this.subTestStats.get(parentTestCaseId); if (subtestStats) { subtestStats.failed += 1; } else { - subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); + this.subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 }); runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); // clear since subtest items don't persist between runs clearAllChildren(parentTestItem); @@ -200,11 +201,11 @@ export class PythonResultResolver implements ITestResultResolver { // find the subtest's parent test item if (parentTestItem) { - const subtestStats = subTestStats.get(parentTestCaseId); + const subtestStats = this.subTestStats.get(parentTestCaseId); if (subtestStats) { subtestStats.passed += 1; } else { - subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); + this.subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 }); runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`)); // clear since subtest items don't persist between runs clearAllChildren(parentTestItem); @@ -228,4 +229,4 @@ export class PythonResultResolver implements ITestResultResolver { } return Promise.resolve(); } -} \ No newline at end of file +} diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 4378c68b534c..211c322d67f1 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -8,7 +8,7 @@ import { SpawnOptions, } from '../../../common/process/types'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred, Deferred } from '../../../common/utils/async'; +import { createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { traceError, traceVerbose } from '../../../logging'; import { @@ -23,8 +23,6 @@ import { * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. #this seems incorrectly copied */ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { - private promiseMap: Map> = new Map(); - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, @@ -55,7 +53,6 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); const uuid = this.testServer.createUUID(uri.fsPath); - this.promiseMap.set(uuid, deferred); const settings = this.configSettings.getSettings(uri); const { pytestArgs } = settings.testing; diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 1bf032b9b594..b68b80945ef3 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -4,7 +4,7 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred, Deferred } from '../../../common/utils/async'; +import { createDeferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; import { DataReceivedEvent, @@ -31,8 +31,6 @@ import { startTestIdServer } from '../common/utils'; */ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { - private promiseMap: Map> = new Map(); - constructor( public testServer: ITestServer, public configSettings: IConfigurationService, @@ -48,6 +46,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { + const uuid = this.testServer.createUUID(uri.fsPath); traceVerbose(uri, testIds, debugBool); const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { @@ -55,18 +54,26 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { } }); try { - await this.runTestsNew(uri, testIds, debugBool, executionFactory, debugLauncher); + await this.runTestsNew(uri, testIds, uuid, debugBool, executionFactory, debugLauncher); } finally { + this.testServer.deleteUUID(uuid); disposable.dispose(); // confirm with testing that this gets called (it must clean this up) } - const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; + // placeholder until after the rewrite is adopted + // TODO: remove after adoption. + const executionPayload: ExecutionTestPayload = { + cwd: uri.fsPath, + status: 'success', + error: '', + }; return executionPayload; } private async runTestsNew( uri: Uri, testIds: string[], + uuid: string, debugBool?: boolean, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, @@ -75,8 +82,6 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); this.configSettings.isTestExecution(); - const uuid = this.testServer.createUUID(uri.fsPath); - this.promiseMap.set(uuid, deferred); const settings = this.configSettings.getSettings(uri); const { pytestArgs } = settings.testing; diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 8d393a8da18d..a9b4c957ec27 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -4,7 +4,6 @@ import * as path from 'path'; import { Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred, Deferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -20,8 +19,6 @@ import { * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. */ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { - private promiseMap: Map> = new Map(); - private cwd: string | undefined; constructor( @@ -32,7 +29,7 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { ) {} public async discoverTests(uri: Uri): Promise { - const deferred = createDeferred(); + // const deferred = createDeferred(); const settings = this.configSettings.getSettings(uri); const { unittestArgs } = settings.testing; @@ -49,18 +46,23 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { outChannel: this.outputChannel, }; - this.promiseMap.set(uuid, deferred); - const disposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); }); try { await this.callSendCommand(options); } finally { + this.testServer.deleteUUID(uuid); disposable.dispose(); + // deferred.resolve(); // confirm with testing that this gets called (it must clean this up) } - const discoveryPayload: DiscoveredTestPayload = { cwd: uri.fsPath, status: 'success' }; + // placeholder until after the rewrite is adopted + // TODO: remove after adoption. + const discoveryPayload: DiscoveredTestPayload = { + cwd: uri.fsPath, + status: 'success', + }; return discoveryPayload; } diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index ca88d3871706..bf3038817f8b 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { Deferred, createDeferred } from '../../../common/utils/async'; +import { createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -23,8 +23,6 @@ import { startTestIdServer } from '../common/utils'; */ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { - private promiseMap: Map> = new Map(); - private cwd: string | undefined; constructor( @@ -40,14 +38,16 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { debugBool?: boolean, runInstance?: TestRun, ): Promise { + const uuid = this.testServer.createUUID(uri.fsPath); const disposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); } }); try { - await this.runTestsNew(uri, testIds, debugBool); + await this.runTestsNew(uri, testIds, uuid, debugBool); } finally { + this.testServer.deleteUUID(uuid); disposable.dispose(); // confirm with testing that this gets called (it must clean this up) } @@ -55,13 +55,17 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { return executionPayload; } - private async runTestsNew(uri: Uri, testIds: string[], debugBool?: boolean): Promise { + private async runTestsNew( + uri: Uri, + testIds: string[], + uuid: string, + debugBool?: boolean, + ): Promise { const settings = this.configSettings.getSettings(uri); const { cwd, unittestArgs } = settings.testing; const command = buildExecutionCommand(unittestArgs); this.cwd = cwd || uri.fsPath; - const uuid = this.testServer.createUUID(uri.fsPath); const options: TestCommandOptions = { workspaceFolder: uri, @@ -74,16 +78,15 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { }; const deferred = createDeferred(); - this.promiseMap.set(uuid, deferred); traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); const runTestIdsPort = await startTestIdServer(testIds); await this.testServer.sendCommand(options, runTestIdsPort.toString(), () => { - // disposable.dispose(); deferred.resolve(); }); - // return deferred.promise; + // placeholder until after the rewrite is adopted + // TODO: remove after adoption. const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; return executionPayload; } From 0d92b85117b7bd89e1e3156a53c540f9bc20499b Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 21 Jun 2023 14:33:52 -0700 Subject: [PATCH 29/34] subtests generating correctly --- pythonFiles/unittestadapter/execution.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 764e6915bff9..1681a5ba1cac 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -148,7 +148,6 @@ def formatResult( "traceback": tb, "subtest": subtest.id() if subtest else None, } - self.formatted[test_id] = result if PORT == 0 or UUID == 0: print("Error sending response, port or uuid unknown to python server.") @@ -229,13 +228,18 @@ def run_tests( def send_run_data(raw_data, port, uuid): - print("sending run data") + print("sending run data1") # Build the request data (it has to be a POST request or the Node side will not process it), and send it. status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) - test_id = raw_data["test"] + if raw_data["subtest"]: + test_id = raw_data["subtest"] + else: + test_id = raw_data["test"] test_dict = {} + print("sending run data2") test_dict[test_id] = raw_data + print("sending run data3") payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} addr = ("localhost", port) data = json.dumps(payload) @@ -244,6 +248,7 @@ def send_run_data(raw_data, port, uuid): Request-uuid: {uuid} {data}""" + print("data to send: ", request) try: with socket_manager.SocketManager(addr) as s: if s.socket is not None: From 7c68e226095077b9a919804a20069d5791189546 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 22 Jun 2023 11:09:06 -0700 Subject: [PATCH 30/34] fix discovery error handling --- pythonFiles/vscode_pytest/__init__.py | 21 ++++++++++++++++--- .../testController/common/resultResolver.ts | 6 +++--- .../testing/testController/common/types.ts | 2 +- .../resultResolver.unit.test.ts | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index de201529fda1..74ff5008fbbd 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -56,7 +56,7 @@ def pytest_internalerror(excrepr, excinfo): excinfo -- the exception information of type ExceptionInfo. """ # call.excinfo.exconly() returns the exception as a string. - ERRORS.append(excinfo.exconly()) + ERRORS.append(excinfo.exconly() + "\n Check Python Test Logs for more details.") def pytest_exception_interact(node, call, report): @@ -70,7 +70,13 @@ def pytest_exception_interact(node, call, report): # call.excinfo is the captured exception of the call, if it raised as type ExceptionInfo. # call.excinfo.exconly() returns the exception as a string. if call.excinfo and call.excinfo.typename != "AssertionError": - ERRORS.append(call.excinfo.exconly()) + ERRORS.append( + call.excinfo.exconly() + "\n Check Python Test Logs for more details." + ) + else: + ERRORS.append( + report.longreprtext + "\n Check Python Test Logs for more details." + ) def pytest_keyboard_interrupt(excinfo): @@ -80,7 +86,7 @@ def pytest_keyboard_interrupt(excinfo): excinfo -- the exception information of type ExceptionInfo. """ # The function execonly() returns the exception as a string. - ERRORS.append(excinfo.exconly()) + ERRORS.append(excinfo.exconly() + "\n Check Python Test Logs for more details.") class TestOutcome(Dict): @@ -200,6 +206,15 @@ def pytest_sessionfinish(session, exitstatus): ) cwd = pathlib.Path.cwd() if IS_DISCOVERY: + if not (exitstatus == 0 or exitstatus == 1 or exitstatus == 5): + errorNode: TestNode = { + "name": "", + "path": cwd, + "type_": "error", + "children": [], + "id_": "", + } + post_response(os.fsdecode(cwd), errorNode) try: session_node: Union[TestNode, None] = build_test_tree(session) if not session_node: diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index c9726cf04f66..9b2ad6abdf78 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -49,13 +49,13 @@ export class PythonResultResolver implements ITestResultResolver { if (rawTestData.status === 'error') { const testingErrorConst = this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery; - const { errors } = rawTestData; - traceError(testingErrorConst, '\r\n', errors!.join('\r\n\r\n')); + const { error } = rawTestData; + traceError(testingErrorConst, '\r\n', error?.join('\r\n\r\n') ?? ''); let errorNode = this.testController.items.get(`DiscoveryError:${workspacePath}`); const message = util.format( `${testingErrorConst} ${Testing.seePythonOutput}\r\n`, - errors!.join('\r\n\r\n'), + error?.join('\r\n\r\n') ?? '', ); if (errorNode === undefined) { diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index cb7fda797c4a..d4e54951bfd7 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -231,7 +231,7 @@ export type DiscoveredTestPayload = { cwd: string; tests?: DiscoveredTestNode; status: 'success' | 'error'; - errors?: string[]; + error?: string[]; }; export type ExecutionTestPayload = { diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts index 57b321c2e36c..94ec1cc10a9a 100644 --- a/src/test/testing/testController/resultResolver.unit.test.ts +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -113,7 +113,7 @@ suite('Result Resolver tests', () => { const payload: DiscoveredTestPayload = { cwd: workspaceUri.fsPath, status: 'error', - errors: [errorMessage], + error: [errorMessage], }; const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { id: 'id', From 37b8b2a9eb653fe3839b98aff8a39de656a00df0 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 22 Jun 2023 11:23:54 -0700 Subject: [PATCH 31/34] fix imports --- pythonFiles/tests/unittestadapter/test_execution.py | 5 +++++ pythonFiles/unittestadapter/execution.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pythonFiles/tests/unittestadapter/test_execution.py b/pythonFiles/tests/unittestadapter/test_execution.py index d461ead9ad94..141fcc68f982 100644 --- a/pythonFiles/tests/unittestadapter/test_execution.py +++ b/pythonFiles/tests/unittestadapter/test_execution.py @@ -3,9 +3,14 @@ import os import pathlib +import sys from typing import List import pytest + +PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +sys.path.insert(0, PYTHON_FILES) from unittestadapter.execution import parse_execution_cli_args, run_tests TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 1681a5ba1cac..1a73f8aa8270 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -22,7 +22,7 @@ sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python")) from testing_tools import process_json_util, socket_manager from typing_extensions import NotRequired, TypeAlias, TypedDict -from utils import parse_unittest_args +from unittestadapter.utils import parse_unittest_args DEFAULT_PORT = "45454" From 96c07e2655027a3fe0efc527031fc3e9c7782dad Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 22 Jun 2023 11:26:05 -0700 Subject: [PATCH 32/34] fix pyright errors --- pythonFiles/tests/pytestadapter/test_discovery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 744683c0cdf7..7f2355129c65 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import os import shutil +from typing import Any, Dict, List, Optional import pytest From 81ba71dd3985a2bb0051eb7ed3e3aede0280777f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 22 Jun 2023 11:32:29 -0700 Subject: [PATCH 33/34] remove extra prints --- pythonFiles/unittestadapter/execution.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 1a73f8aa8270..2f94b54992aa 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -228,7 +228,6 @@ def run_tests( def send_run_data(raw_data, port, uuid): - print("sending run data1") # Build the request data (it has to be a POST request or the Node side will not process it), and send it. status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) @@ -237,9 +236,7 @@ def send_run_data(raw_data, port, uuid): else: test_id = raw_data["test"] test_dict = {} - print("sending run data2") test_dict[test_id] = raw_data - print("sending run data3") payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} addr = ("localhost", port) data = json.dumps(payload) @@ -248,7 +245,6 @@ def send_run_data(raw_data, port, uuid): Request-uuid: {uuid} {data}""" - print("data to send: ", request) try: with socket_manager.SocketManager(addr) as s: if s.socket is not None: From ad390d7f84fa057089b1229f02d852d6135633bd Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 23 Jun 2023 14:48:56 -0700 Subject: [PATCH 34/34] small edits for review --- pythonFiles/tests/unittestadapter/test_execution.py | 4 ++-- pythonFiles/unittestadapter/discovery.py | 4 ++-- pythonFiles/unittestadapter/execution.py | 4 ++-- pythonFiles/vscode_pytest/__init__.py | 1 - .../testing/testController/unittest/testDiscoveryAdapter.ts | 3 --- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pythonFiles/tests/unittestadapter/test_execution.py b/pythonFiles/tests/unittestadapter/test_execution.py index 141fcc68f982..c7a5cb0eef61 100644 --- a/pythonFiles/tests/unittestadapter/test_execution.py +++ b/pythonFiles/tests/unittestadapter/test_execution.py @@ -8,9 +8,9 @@ import pytest -PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PYTHON_FILES = pathlib.Path(__file__).parent.parent -sys.path.insert(0, PYTHON_FILES) +sys.path.insert(0, os.fspath(PYTHON_FILES)) from unittestadapter.execution import parse_execution_cli_args, run_tests TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py index bcc2fd967f78..02490024b217 100644 --- a/pythonFiles/unittestadapter/discovery.py +++ b/pythonFiles/unittestadapter/discovery.py @@ -17,8 +17,8 @@ from typing_extensions import Literal # Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. -PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, PYTHON_FILES) +PYTHON_FILES = pathlib.Path(__file__).parent.parent +sys.path.insert(0, os.fspath(PYTHON_FILES)) from testing_tools import socket_manager diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 2f94b54992aa..d9d5f41d624b 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -15,9 +15,9 @@ directory_path = pathlib.Path(__file__).parent.parent / "lib" / "python" # Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. -PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PYTHON_FILES = pathlib.Path(__file__).parent.parent -sys.path.insert(0, PYTHON_FILES) +sys.path.insert(0, os.fspath(PYTHON_FILES)) # Add the lib path to sys.path to find the typing_extensions module. sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python")) from testing_tools import process_json_util, socket_manager diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 74ff5008fbbd..1178abdb93d6 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -173,7 +173,6 @@ def pytest_report_teststatus(report, config): "success", collected_test if collected_test else None, ) - # collected_tests[report.nodeid] = item_result ERROR_MESSAGE_CONST = { diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index a9b4c957ec27..69438e341f14 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -29,7 +29,6 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { ) {} public async discoverTests(uri: Uri): Promise { - // const deferred = createDeferred(); const settings = this.configSettings.getSettings(uri); const { unittestArgs } = settings.testing; @@ -54,8 +53,6 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { } finally { this.testServer.deleteUUID(uuid); disposable.dispose(); - // deferred.resolve(); - // confirm with testing that this gets called (it must clean this up) } // placeholder until after the rewrite is adopted // TODO: remove after adoption.