test: add E2E for multi workspace scenario

e2e/helpers/constants.ts

@@ -1,3 +1,4 @@
+import * as cucumber from './cucumber.ts'
 import { STATUS } from './index.ts'
 
 function createExpectedNotRun(targetFramework: 'mocha' | 'jasmine') {
@@ -90,3 +91,59 @@ export function createExpected(targetFramework: 'mocha' | 'jasmine') {
         runPartially: [createExpectedRunPartially(targetFramework)],
     }
 }
+
+export function createWorkspaceExpected() {
+    return {
+        notRun: [
+            {
+                text: 'cucumber',
+                status: STATUS.NOT_YET_RUN,
+                children: [cucumber.createExpectedNotRun()],
+            },
+            {
+                text: 'jasmine',
+                status: STATUS.NOT_YET_RUN,
+                children: [createExpectedNotRun('jasmine')],
+            },
+            {
+                text: 'mocha',
+                status: STATUS.NOT_YET_RUN,
+                children: [createExpectedNotRun('mocha')],
+            },
+        ],
+        runAll: [
+            {
+                text: 'cucumber',
+                status: STATUS.PASSED,
+                children: [cucumber.createExpectedRunAll()],
+            },
+            {
+                text: 'jasmine',
+                status: STATUS.FAILED,
+                children: [createExpectedRunAll('jasmine')],
+            },
+            {
+                text: 'mocha',
+                status: STATUS.FAILED,
+                children: [createExpectedRunAll('mocha')],
+            },
+        ],
+        runPartially: [
+            {
+                text: 'cucumber',
+                status: STATUS.NOT_YET_RUN,
+                children: [cucumber.createExpectedNotRun()],
+            },
+            {
+                text: 'jasmine',
+                status: STATUS.PASSED,
+                children: [createExpectedRunPartially('jasmine')],
+            },
+            {
+                text: 'mocha',
+                status: STATUS.NOT_YET_RUN,
+                children: [createExpectedNotRun('mocha')],
+            },
+        ],
+    }
+}

e2e/helpers/index.ts

@@ -115,3 +115,12 @@ export async function clickTitleActionButton(titlePart: ViewTitlePart, label: st
         }
     }
 }
+
+export async function collapseAllTests(testingSection: DefaultTreeSection) {
+    const items = (await testingSection.getVisibleItems()).reverse()
+    for (const item of items) {
+        if ((await item.isExpandable()) && (await item.isExpanded())) {
+            await item.collapse()
+        }
+    }
+}

e2e/package.json

@@ -14,6 +14,7 @@
         "test:e2e:mocha": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=mocha xvfb-maybe pnpm run wdio",
         "test:e2e:jasmine": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=jasmine xvfb-maybe pnpm run wdio",
         "test:e2e:cucumber": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=cucumber xvfb-maybe pnpm run wdio",
+        "test:e2e:workspace": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=workspace xvfb-maybe pnpm run wdio",
         "test:smoke": "run-s test:smoke:*",
         "test:smoke:update-config": "xvfb-maybe wdio run ./wdioSmoke.conf.ts",
         "wdio": "wdio run ./wdio.conf.ts"
@@ -25,6 +26,7 @@
         "@wdio/local-runner": "^9.13.0",
         "@wdio/mocha-framework": "^9.13.0",
         "@wdio/spec-reporter": "^9.13.0",
+        "@wdio/types": "^9.15.0",
         "chai": "^5.2.0",
         "expect": "^29.7.0",
         "semver": "^7.7.2",

e2e/tests/basic.spec.ts

@@ -6,6 +6,7 @@ import {
     clearAllTestResults,
     clickTitleActionButton,
     clickTreeItemButton,
+    collapseAllTests,
     getTestingSection,
     openTestingView,
     waitForResolved,
@@ -30,12 +31,8 @@ describe(`VS Code Extension Testing with ${targetFramework}`, function () {
         sideBarView = workbench.getSideBar()
 
         const testingSection = await getTestingSection(sideBarView.getContent())
-        const items = (await testingSection.getVisibleItems()).reverse()
-        for (const item of items) {
-            if ((await item.isExpandable()) && (await item.isExpanded())) {
-                await item.collapse()
-            }
-        }
+        await collapseAllTests(testingSection)
+
         await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
     })
 

e2e/tests/basicCucumber.spec.ts

@@ -6,6 +6,7 @@ import {
     clearAllTestResults,
     clickTitleActionButton,
     clickTreeItemButton,
+    collapseAllTests,
     getTestingSection,
     openTestingView,
     waitForResolved,
@@ -30,12 +31,8 @@ describe(`VS Code Extension Testing with ${targetFramework}`, function () {
         sideBarView = workbench.getSideBar()
 
         const testingSection = await getTestingSection(sideBarView.getContent())
-        const items = (await testingSection.getVisibleItems()).reverse()
-        for (const item of items) {
-            if ((await item.isExpandable()) && (await item.isExpanded())) {
-                await item.collapse()
-            }
-        }
+        await collapseAllTests(testingSection)
+
         await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
     })
 

e2e/tests/basicWorkspace.spec.ts

@@ -0,0 +1,117 @@
+import { browser, expect } from '@wdio/globals'
+import { createWorkspaceExpected } from 'helpers/constants.ts'
+
+import {
+    STATUS,
+    clearAllTestResults,
+    collapseAllTests,
+    clickTitleActionButton,
+    clickTreeItemButton,
+    getTestingSection,
+    openTestingView,
+    waitForResolved,
+    waitForTestStatus,
+} from '../helpers/index.ts'
+import type { SideBarView, ViewControl, Workbench } from 'wdio-vscode-service'
+
+const expected = createWorkspaceExpected()
+
+describe('VS Code Extension Testing with Workspace', function () {
+    this.retries(3)
+    let workbench: Workbench
+    let testingViewControl: ViewControl
+    let sideBarView: SideBarView<any>
+
+    beforeEach(async function () {
+        workbench = await browser.getWorkbench()
+        testingViewControl = await openTestingView(workbench)
+        sideBarView = workbench.getSideBar()
+
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        await collapseAllTests(testingSection)
+        await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 3)
+    })
+
+    afterEach(async function () {
+        await clearAllTestResults(workbench)
+    })
+
+    it('should be displayed the testing screen at the sideBar', async function () {
+        expect(await testingViewControl.getTitle()).toBe('Testing')
+        expect(await sideBarView.getTitlePart().getTitle()).toBe('TESTING')
+    })
+
+    it('should resolve defined tests correctly', async function () {
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        const items = await testingSection.getVisibleItems()
+
+        await waitForResolved(browser, items[0])
+
+        expect(items.length).toBe(expected.notRun.length)
+
+        /**
+         * Because of the small screen size of the CI environment,
+         * the tree is expanded and asserted per workspace.
+         */
+        for (let index = 0; index < expected.notRun.length; index++) {
+            await expect([items[index]]).toMatchTreeStructure([expected.notRun[index]])
+
+            await collapseAllTests(testingSection)
+        }
+    })
+
+    it('should run at top Level', async function () {
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        const items = await testingSection.getVisibleItems()
+
+        await waitForResolved(browser, items[0])
+
+        await clickTitleActionButton(sideBarView.getTitlePart(), 'Run Tests')
+
+        expect(items.length).toBe(expected.runAll.length)
+
+        await waitForTestStatus(browser, items[0], STATUS.PASSED)
+        await waitForTestStatus(browser, items[1], STATUS.FAILED)
+        await waitForTestStatus(browser, items[2], STATUS.FAILED)
+
+        /**
+         * Because of the small screen size of the CI environment,
+         * the tree is expanded and asserted per workspace.
+         */
+        for (let index = 0; index < expected.runAll.length; index++) {
+            await expect([items[index]]).toMatchTreeStructure([expected.runAll[index]])
+
+            await collapseAllTests(testingSection)
+        }
+    })
+
+    it('should run at not top Level', async function () {
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        const items = await testingSection.getVisibleItems()
+
+        await waitForResolved(browser, items[0])
+        await waitForResolved(browser, items[0])
+
+        const target = await items[1]
+            .getChildren()
+            .then((items) => items[0].getChildren())
+            .then((items) => items[0].getChildren())
+            .then((items) => items[0].getChildren())
+            .then((items) => items[1])
+
+        await clickTreeItemButton(browser, target, 'Run Test')
+
+        expect(items.length).toBe(expected.runPartially.length)
+        await waitForTestStatus(browser, items[1], STATUS.PASSED)
+
+        /**
+         * Because of the small screen size of the CI environment,
+         * the tree is expanded and asserted per workspace.
+         */
+        for (let index = 0; index < expected.runPartially.length; index++) {
+            await expect([items[index]]).toMatchTreeStructure([expected.runPartially[index]])
+
+            await collapseAllTests(testingSection)
+        }
+    })
+})

e2e/tests/updateConfig.spec.ts

@@ -8,6 +8,7 @@ import {
     STATUS,
     clearAllTestResults,
     clickTreeItemButton,
+    collapseAllTests,
     getTestingSection,
     openTestingView,
     waitForResolved,
@@ -32,12 +33,8 @@ describe('VS Code Extension Testing (Update config)', function () {
         sideBarView = workbench.getSideBar()
 
         const testingSection = await getTestingSection(sideBarView.getContent())
-        const items = (await testingSection.getVisibleItems()).reverse()
-        for (const item of items) {
-            if ((await item.isExpandable()) && (await item.isExpanded())) {
-                await item.collapse()
-            }
-        }
+        await collapseAllTests(testingSection)
+
         await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
     })
 

e2e/tests/updateSpec.spec.ts

@@ -8,6 +8,7 @@ import {
     STATUS,
     clearAllTestResults,
     clickTreeItemButton,
+    collapseAllTests,
     getTestingSection,
     openTestingView,
     waitForResolved,
@@ -36,12 +37,8 @@ describe('VS Code Extension Testing (Update config)', function () {
         sideBarView = workbench.getSideBar()
 
         const testingSection = await getTestingSection(sideBarView.getContent())
-        const items = (await testingSection.getVisibleItems()).reverse()
-        for (const item of items) {
-            if ((await item.isExpandable()) && (await item.isExpanded())) {
-                await item.collapse()
-            }
-        }
+        await collapseAllTests(testingSection)
+
         await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
     })
 

e2e/wdio.conf.ts

@@ -4,9 +4,12 @@ import * as url from 'node:url'
 import { minVersion } from 'semver'
 
 import pkg from '../packages/vscode-webdriverio/package.json' with { type: 'json' }
+import type { Frameworks } from '@wdio/types'
+
+type TestTargets = 'workspace' | 'mocha' | 'jasmine' | 'cucumber'
 
 const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
-const target = process.env.VSCODE_WDIO_E2E_FRAMEWORK || 'mocha'
+const target = (process.env.VSCODE_WDIO_E2E_FRAMEWORK || 'mocha') as TestTargets
 
 const minimumVersion = minVersion(pkg.engines.vscode)?.version || 'stable'
 
@@ -16,7 +19,19 @@ const version = isCompatibilityMode ? minimumVersion : 'stable'
 const outputDir = path.join(__dirname, 'logs', [isCompatibilityMode ? 'compatibility' : 'e2e', target].join('-'))
 process.env.VSCODE_WDIO_TRACE_LOG_PATH = outputDir
 
-const specs = target === 'cucumber' ? ['./tests/basicCucumber.spec.ts'] : ['./tests/basic.spec.ts']
+function defineSpecs(target: TestTargets) {
+    switch (target) {
+        case 'cucumber':
+            return ['./tests/basicCucumber.spec.ts']
+        case 'workspace':
+            return ['./tests/basicWorkspace.spec.ts']
+        default:
+            return ['./tests/basic.spec.ts']
+    }
+}
+
+const specs = defineSpecs(target)
+let screenshotCount = 0
 
 export function createBaseConfig(workspacePath: string): WebdriverIO.Config {
     return {
@@ -55,10 +70,17 @@ export function createBaseConfig(workspacePath: string): WebdriverIO.Config {
             timeout: 6000000,
             require: ['assertions/index.ts'],
         },
+        afterTest: async function (_test:unknown, _context:unknown, result: Frameworks.TestResult) {
+            if (!result.passed) {
+                await browser.saveScreenshot(path.join(outputDir, `screenshot-${screenshotCount++}.png`))
+            }
+        },
     }
 }
 
+const workspace = target === 'workspace' ? '../samples/e2e/wdio.code-workspace' : `../samples/e2e/${target}`
+
 export const config: WebdriverIO.Config = {
-    ...createBaseConfig(`../samples/e2e/${target}`),
+    ...createBaseConfig(workspace),
     specs,
 }

samples/e2e/cucumber/wdio.conf.ts

@@ -52,6 +52,9 @@ export const config: WebdriverIO.Config = {
     capabilities: [
         {
             browserName: 'chrome',
+            'goog:chromeOptions': {
+                args: ['headless', 'disable-gpu'],
+            },
         },
     ],
 

samples/e2e/jasmine/wdio.conf.ts

@@ -9,6 +9,9 @@ export const config: WebdriverIO.Config = {
     capabilities: [
         {
             browserName: 'chrome',
+            'goog:chromeOptions': {
+                args: ['headless', 'disable-gpu'],
+            },
         },
     ],
 

samples/e2e/mocha/wdio.conf.ts

@@ -9,6 +9,9 @@ export const config: WebdriverIO.Config = {
     capabilities: [
         {
             browserName: 'chrome',
+            'goog:chromeOptions': {
+                args: ['headless', 'disable-gpu'],
+            },
         },
     ],
     logLevel: 'info',