From ed3551b7be46eeaa3f90b5efbc975b701d943430 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 16 Apr 2022 14:08:01 +0200 Subject: [PATCH] cli: added experimental test configuration and module loader Signed-off-by: Patrik Oldsberg --- .changeset/fast-cars-rest.md | 7 ++ .github/workflows/ci.yml | 2 + .github/workflows/deploy_packages.yml | 1 + .github/workflows/verify_windows.yml | 1 + packages/cli/config/jest.js | 15 +++- .../cli/config/jestCachingModuleLoader.js | 81 +++++++++++++++++++ packages/cli/package.json | 1 + packages/cli/src/lib/parallel.ts | 2 + 8 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 .changeset/fast-cars-rest.md create mode 100644 packages/cli/config/jestCachingModuleLoader.js diff --git a/.changeset/fast-cars-rest.md b/.changeset/fast-cars-rest.md new file mode 100644 index 0000000000..4996da96fa --- /dev/null +++ b/.changeset/fast-cars-rest.md @@ -0,0 +1,7 @@ +--- +'@backstage/cli': patch +--- + +Introduced a new experimental test configuration with a number of changes. It switches the coverage provider from `v8` to the default Babel provider, along with always enabling source maps in the Sucrase transform. It also adds a custom module loader that caches both file transforms and VM script objects across all projects in a test run, which provides a big performance boost when running tests from the project root, increasing speed and reducing memory usage. + +This new configuration is not enabled by default. It is enabled by setting the environment variable `BACKSTAGE_NEXT_TESTS` to a non-empty value. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 796df14ac7..31a315d044 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,7 @@ jobs: if: ${{ steps.yarn-lock.outcome == 'success' }} run: yarn lerna -- run test --since origin/master -- --coverage --runInBand env: + BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} @@ -166,6 +167,7 @@ jobs: yarn lerna -- run test -- --coverage --runInBand bash <(curl -s https://codecov.io/bash) -N $(git rev-parse FETCH_HEAD) env: + BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index fd9ed216be..937981ae6c 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -122,6 +122,7 @@ jobs: bash <(curl -s https://codecov.io/bash) -f packages/core-components/coverage/* -F core-components bash <(curl -s https://codecov.io/bash) -f packages/core-plugin-api/coverage/* -F core-plugin-api env: + BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 6899b6ec2b..7156bec4f6 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -58,6 +58,7 @@ jobs: - name: test run: yarn lerna -- run test env: + BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 # credit: https://github.com/appleboy/discord-action/issues/3#issuecomment-731426861 diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index 6b9201559b..e5e08b4546 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -20,6 +20,11 @@ const crypto = require('crypto'); const glob = require('util').promisify(require('glob')); const { version } = require('../package.json'); +const envOptions = { + nextTests: Boolean(process.env.BACKSTAGE_NEXT_TESTS), + enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS), +}; + const transformIgnorePattern = [ '@material-ui', '@rjsf', @@ -118,7 +123,7 @@ async function getProjectConfig(targetPath, displayName) { ...(displayName && { displayName }), rootDir: path.resolve(targetPath, 'src'), coverageDirectory: path.resolve(targetPath, 'coverage'), - coverageProvider: 'v8', + coverageProvider: envOptions.nextTests ? undefined : 'v8', collectCoverageFrom: ['**/*.{js,jsx,ts,tsx,mjs,cjs}', '!**/*.d.ts'], moduleNameMapper: { '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), @@ -127,7 +132,9 @@ async function getProjectConfig(targetPath, displayName) { transform: { '\\.(js|jsx|ts|tsx|mjs|cjs)$': [ require.resolve('./jestSucraseTransform.js'), - { enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS) }, + { + enableSourceMaps: envOptions.enableSourceMaps || envOptions.nextTests, + }, ], '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg|eot|woff|woff2|ttf)$': require.resolve('./jestFileTransform.js'), @@ -137,6 +144,10 @@ async function getProjectConfig(targetPath, displayName) { // A bit more opinionated testMatch: ['**/*.test.{js,jsx,ts,tsx,mjs,cjs}'], + moduleLoader: envOptions.nextTests + ? require.resolve('./jestCachingModuleLoader') + : undefined, + transformIgnorePatterns: [`/node_modules/(?:${transformIgnorePattern})/`], ...getRoleConfig(closestPkgJson?.backstage?.role), diff --git a/packages/cli/config/jestCachingModuleLoader.js b/packages/cli/config/jestCachingModuleLoader.js new file mode 100644 index 0000000000..9ef29d7485 --- /dev/null +++ b/packages/cli/config/jestCachingModuleLoader.js @@ -0,0 +1,81 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const { default: JestRuntime } = require('jest-runtime'); + +const fileTransformCache = new Map(); +const scriptTransformCache = new Map(); + +let runtimeGeneration = 0; + +module.exports = class CachingJestRuntime extends JestRuntime { + // Each Jest run creates a new runtime, including when rerunning tests in + // watch mode. This keeps track of whether we've switched runtime instance. + __runtimeGeneration = runtimeGeneration++; + + transformFile(filename, options) { + const entry = fileTransformCache.get(filename); + if (entry) { + // Only check modification time if it's from a different runtime generation + if (entry.generation === this.__runtimeGeneration) { + return entry.code; + } + + // Keep track of the modification time of files so that we can properly + // reprocess them in watch mode. + const { mtimeMs } = fs.statSync(filename); + if (mtimeMs > entry.mtimeMs) { + const code = super.transformFile(filename, options); + fileTransformCache.set(filename, { + code, + mtimeMs, + generation: this.__runtimeGeneration, + }); + return code; + } + + fileTransformCache.set(filename, { + ...entry, + generation: this.__runtimeGeneration, + }); + return entry.code; + } + + const code = super.transformFile(filename, options); + fileTransformCache.set(filename, { + code, + mtimeMs: fs.statSync(filename).mtimeMs, + generation: this.__runtimeGeneration, + }); + return code; + } + + // This may or may not be a good idea. Theoretically I don't know why this would impact + // test correctness and flakiness, but it seems like it may introduce flakiness and strange failures. + // It does seem to speed up test execution by a fair amount though. + createScriptFromCode(scriptSource, filename) { + let script = scriptTransformCache.get(scriptSource); + if (!script) { + script = super.createScriptFromCode(scriptSource, filename); + // Tried to store the script object in a WeakRef here. It starts out at + // about 90% hit rate, but eventually drops all the way to 20%, and overall + // it seemed to increase memory usage by 20% or so. + scriptTransformCache.set(scriptSource, script); + } + return script; + } +}; diff --git a/packages/cli/package.json b/packages/cli/package.json index 24f9689374..305e84a5ad 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -86,6 +86,7 @@ "html-webpack-plugin": "^5.3.1", "inquirer": "^8.2.0", "jest": "^27.5.1", + "jest-runtime": "^27.5.1", "jest-css-modules": "^2.1.0", "jest-transform-yaml": "^1.0.0", "json-schema": "^0.4.0", diff --git a/packages/cli/src/lib/parallel.ts b/packages/cli/src/lib/parallel.ts index 7b4ae4609e..ab32de992c 100644 --- a/packages/cli/src/lib/parallel.ts +++ b/packages/cli/src/lib/parallel.ts @@ -213,6 +213,7 @@ export async function runWorkerQueueThreads( return results; } +/* istanbul ignore next */ function workerQueueThread( workerFuncFactory: ( data: unknown, @@ -313,6 +314,7 @@ export async function runWorkerThreads( ); } +/* istanbul ignore next */ function workerThread( workerFunc: ( data: unknown,