diff --git a/.changeset/cli-parallel-build-options.md b/.changeset/cli-parallel-build-options.md new file mode 100644 index 0000000000..95f9360738 --- /dev/null +++ b/.changeset/cli-parallel-build-options.md @@ -0,0 +1,12 @@ +--- +'@backstage/cli': minor +--- + +Adds a new `BACKSTAGE_CLI_BUILD_PARELLEL` environment variable to control +parallelism for some build steps. + +This is useful in CI to help avoid out of memory issues when using `terser`. The +`BACKSTAGE_CLI_BUILD_PARELLEL` environment variable can be set to +`true | false | [integer]` to override the default behaviour. See +[terser-webpack-plugin](https://github.com/webpack-contrib/terser-webpack-plugin#parallel) +for more details. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index c20a9862b0..6a2eecdee0 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -196,6 +196,7 @@ validators Voi Wealthsimple Weaveworks +Webpack xyz yaml Zalando diff --git a/packages/cli/package.json b/packages/cli/package.json index 5bd732f1d2..084254d7ff 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -86,6 +86,7 @@ "style-loader": "^1.2.1", "sucrase": "^3.14.1", "tar": "^6.0.1", + "terser-webpack-plugin": "^1.4.3", "ts-jest": "^26.0.0", "ts-loader": "^7.0.4", "typescript": "^3.9.3", diff --git a/packages/cli/src/commands/app/build.ts b/packages/cli/src/commands/app/build.ts index 9713acd233..ae67616515 100644 --- a/packages/cli/src/commands/app/build.ts +++ b/packages/cli/src/commands/app/build.ts @@ -19,6 +19,7 @@ import { loadConfig } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from '../../lib/paths'; import { buildBundle } from '../../lib/bundler'; +import { parseParallel, PARALLEL_ENV_VAR } from '../../lib/parallel'; export default async (cmd: Command) => { const appConfigs = await loadConfig({ @@ -27,6 +28,7 @@ export default async (cmd: Command) => { }); await buildBundle({ entry: 'src/index', + parallel: parseParallel(process.env[PARALLEL_ENV_VAR]), statsJsonEnabled: cmd.stats, config: ConfigReader.fromConfigs(appConfigs), appConfigs, diff --git a/packages/cli/src/commands/backend/buildImage.ts b/packages/cli/src/commands/backend/buildImage.ts index b6e9da7b39..b6ae0dd579 100644 --- a/packages/cli/src/commands/backend/buildImage.ts +++ b/packages/cli/src/commands/backend/buildImage.ts @@ -14,12 +14,13 @@ * limitations under the License. */ +import { Command } from 'commander'; import fs from 'fs-extra'; import { join as joinPath, relative as relativePath } from 'path'; import { createDistWorkspace } from '../../lib/packager'; import { paths } from '../../lib/paths'; import { run } from '../../lib/run'; -import { Command } from 'commander'; +import { parseParallel, PARALLEL_ENV_VAR } from '../../lib/parallel'; const PKG_PATH = 'package.json'; @@ -41,6 +42,7 @@ export default async (cmd: Command) => { ...appConfigs, { src: paths.resolveTarget('Dockerfile'), dest: 'Dockerfile' }, ], + parallel: parseParallel(process.env[PARALLEL_ENV_VAR]), skeleton: 'skeleton.tar', }); console.log(`Dist workspace ready at ${tempDistWorkspace}`); diff --git a/packages/cli/src/lib/bundler/optimization.ts b/packages/cli/src/lib/bundler/optimization.ts index acd6a937d9..665eb10b21 100644 --- a/packages/cli/src/lib/bundler/optimization.ts +++ b/packages/cli/src/lib/bundler/optimization.ts @@ -15,7 +15,9 @@ */ import { Options } from 'webpack'; +import TerserPlugin from 'terser-webpack-plugin'; import { BundlingOptions } from './types'; +import { isParallelDefault } from '../parallel'; export const optimization = ( options: BundlingOptions, @@ -24,6 +26,16 @@ export const optimization = ( return { minimize: !isDev, + // Only configure when parallel is explicitly overriden from the default + ...(!isParallelDefault(options.parallel) + ? { + minimizer: [ + new TerserPlugin({ + parallel: options.parallel, + }), + ], + } + : {}), runtimeChunk: 'single', splitChunks: { automaticNameDelimiter: '-', diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index 03d68d9bb8..827d517fef 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -16,6 +16,7 @@ import { AppConfig, Config } from '@backstage/config'; import { BundlingPathsOptions } from './paths'; +import { ParallelOption } from '../parallel'; export type BundlingOptions = { checksEnabled: boolean; @@ -23,6 +24,7 @@ export type BundlingOptions = { config: Config; appConfigs: AppConfig[]; baseUrl: URL; + parallel?: ParallelOption; }; export type BackendBundlingOptions = Omit & { @@ -37,6 +39,7 @@ export type ServeOptions = BundlingPathsOptions & { export type BuildOptions = BundlingPathsOptions & { statsJsonEnabled: boolean; + parallel?: ParallelOption; config: Config; appConfigs: AppConfig[]; }; diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 564317e46a..56f1ee453c 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -20,10 +20,11 @@ import { resolve as resolvePath, relative as relativePath, } from 'path'; +import { tmpdir } from 'os'; +import tar, { CreateOptions } from 'tar'; import { paths } from '../paths'; import { run } from '../run'; -import tar, { CreateOptions } from 'tar'; -import { tmpdir } from 'os'; +import { ParallelOption } from '../parallel'; type LernaPackage = { name: string; @@ -58,6 +59,11 @@ type Options = { */ buildDependencies?: boolean; + /** + * Enable (true/false) or control amount of (number) parallelism in some build steps. + */ + parallel?: ParallelOption; + /** * If set, creates a skeleton tarball that contains all package.json files * with the same structure as the workspace dir. @@ -85,7 +91,12 @@ export async function createDistWorkspace( if (options.buildDependencies) { const scopeArgs = targets.flatMap(target => ['--scope', target.name]); - await run('yarn', ['lerna', 'run', ...scopeArgs, 'build'], { + const lernaArgs = + options.parallel && Number.isInteger(options.parallel) + ? ['--concurrency', options.parallel.toString()] + : []; + + await run('yarn', ['lerna', ...lernaArgs, 'run', ...scopeArgs, 'build'], { cwd: paths.targetRoot, }); } diff --git a/packages/cli/src/lib/parallel.test.ts b/packages/cli/src/lib/parallel.test.ts new file mode 100644 index 0000000000..c8173603d9 --- /dev/null +++ b/packages/cli/src/lib/parallel.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isParallelDefault, parseParallel } from './parallel'; + +describe('parallel', () => { + describe(parseParallel, () => { + it('coerces "false" string to boolean', () => { + expect(parseParallel('false')).toBeFalsy(); + }); + + it('coerces "true" to boolean', () => { + expect(parseParallel('true')).toBeTruthy(); + }); + + it('coerces number string to number', () => { + expect(parseParallel('2')).toBe(2); + }); + it.each([[true], [false], [2]])('returns itself for %p', value => { + expect(parseParallel(value as any)).toEqual(value); + }); + + it.each([[undefined], [null]])('returns true for %p', value => { + expect(parseParallel(value as any)).toBe(true); + }); + + it.each([['on'], [2.5], ['2.5']])('throws error for %p', value => { + expect(() => parseParallel(value as any)).toThrowError( + `Parallel option value '${value}' is not a boolean or integer`, + ); + }); + }); + + describe(isParallelDefault, () => { + it('returns true if default value', () => { + expect(isParallelDefault(undefined)).toBeTruthy(); + expect(isParallelDefault(true)).toBeTruthy(); + }); + + it('returns false if not default value', () => { + expect(isParallelDefault(false)).toBeFalsy(); + expect(isParallelDefault(2)).toBeFalsy(); + expect(isParallelDefault('true' as any)).toBeFalsy(); + }); + }); +}); diff --git a/packages/cli/src/lib/parallel.ts b/packages/cli/src/lib/parallel.ts new file mode 100644 index 0000000000..b6926115aa --- /dev/null +++ b/packages/cli/src/lib/parallel.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const PARALLEL_ENV_VAR = 'BACKSTAGE_CLI_BUILD_PARALLEL'; + +export type ParallelOption = boolean | number | undefined; + +export function isParallelDefault(parallel: ParallelOption) { + return parallel === undefined || parallel === true; +} + +export function parseParallel( + parallel: boolean | string | number | undefined, +): ParallelOption { + if (parallel === undefined || parallel === null) { + return true; + } else if (typeof parallel === 'boolean') { + return parallel; + } else if (typeof parallel === 'number' && Number.isInteger(parallel)) { + return parallel; + } else if (typeof parallel === 'string') { + if (parallel === 'true') { + return true; + } else if (parallel === 'false') { + return false; + } else if (Number.isInteger(parseFloat(parallel.toString()))) { + return Number(parallel); + } + } + + throw Error( + `Parallel option value '${parallel}' is not a boolean or integer`, + ); +} diff --git a/packages/cli/src/types.d.ts b/packages/cli/src/types.d.ts index 94136f1dbc..828819ad17 100644 --- a/packages/cli/src/types.d.ts +++ b/packages/cli/src/types.d.ts @@ -29,3 +29,5 @@ declare module '@svgr/rollup' { } declare module '@rollup/plugin-yaml'; + +declare module 'terser-webpack-plugin';