diff --git a/.changeset/small-donkeys-attack.md b/.changeset/small-donkeys-attack.md new file mode 100644 index 0000000000..97a65cbfd9 --- /dev/null +++ b/.changeset/small-donkeys-attack.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Added a new `--cache [path]` option to the `backstage-cli repo lint` command. The cache keeps track of successful lint runs and avoids re-running linting of individual packages if they haven't changed. This option is primarily intended to be used in CI. diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index 70dc2614cc..64348a872e 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -446,6 +446,7 @@ Usage: backstage-cli repo lint [options] Options: --format --since + --cache [path] --fix -h, --help ``` diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index b54536cd55..9ae52a4972 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -61,6 +61,10 @@ export function registerRepoCommand(program: Command) { '--since ', 'Only lint packages that changed since the specified ref', ) + .option( + '--cache [path]', + 'Enable caching, storing it in node_modules/.cache/backstage-cli by default, or at the provided directory', + ) .option('--fix', 'Attempt to automatically fix violations') .action(lazy(() => import('./repo/lint').then(m => m.command))); diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index d0d6ee8f06..a6381d3520 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -16,7 +16,9 @@ import chalk from 'chalk'; import { Command, OptionValues } from 'commander'; -import { relative as relativePath } from 'path'; +import fs from 'fs-extra'; +import { createHash } from 'crypto'; +import { relative as relativePath, resolve as resolvePath } from 'path'; import { PackageGraph, BackstagePackageJson } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; @@ -30,11 +32,42 @@ function depCount(pkg: BackstagePackageJson) { return deps + devDeps; } +const CACHE_FILE_NAME = 'lint-cache.json'; + +type Cache = string[]; + +async function readCache(dir: string): Promise { + try { + const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); + if (!Array.isArray(data)) { + return undefined; + } + if (data.some(x => typeof x !== 'string')) { + return undefined; + } + return data as Cache; + } catch { + return undefined; + } +} + +async function writeCache(dir: string, cache: Cache) { + await fs.mkdirp(dir); + await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); +} + export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); + const cacheDir = + opts.cache === true + ? paths.resolveTargetRoot('node_modules/.cache/backstage-cli') + : opts.cache; + const cache = cacheDir ? await readCache(cacheDir) : undefined; + + const graph = PackageGraph.fromPackages(packages); + if (opts.since) { - const graph = PackageGraph.fromPackages(packages); packages = await graph.listChangedPackages({ ref: opts.since, analyzeLockfile: true, @@ -57,26 +90,61 @@ export async function command(opts: OptionValues, cmd: Command): Promise { const parseLintScript = createScriptOptionsParser(cmd, ['package', 'lint']); + const items = await Promise.all( + packages.map(async pkg => { + const base = { + fullDir: pkg.dir, + relativeDir: relativePath(paths.targetRoot, pkg.dir), + lintOptions: parseLintScript(pkg.packageJson.scripts?.lint), + parentHash: undefined, + }; + + if (!cacheDir) { + return base; + } + + const hash = createHash('sha1'); + + hash.update(await graph.getDependencyHash(pkg.packageJson.name)); + hash.update('\0'); + hash.update(process.version); // Node.js version + hash.update('\0'); + hash.update('v1'); // The version of this implementation + + return { + ...base, + parentHash: hash.digest('hex'), + }; + }), + ); + const resultsList = await runWorkerQueueThreads({ - items: packages.map(pkg => ({ - fullDir: pkg.dir, - relativeDir: relativePath(paths.targetRoot, pkg.dir), - lintOptions: parseLintScript(pkg.packageJson.scripts?.lint), - })), + items, workerData: { fix: Boolean(opts.fix), format: opts.format as string | undefined, + shouldCache: Boolean(cacheDir), + successCache: cache, }, - workerFactory: async ({ fix, format }) => { + workerFactory: async ({ fix, format, shouldCache, successCache }) => { const { ESLint } = require('eslint') as typeof import('eslint'); + const crypto = require('crypto') as typeof import('crypto'); + const recursiveReadDir = + require('recursive-readdir') as typeof import('recursive-readdir'); + const { readFile } = + require('fs/promises') as typeof import('fs/promises'); + const { relative: workerRelativePath } = + require('path') as typeof import('path'); return async ({ fullDir, relativeDir, lintOptions, + parentHash, }): Promise<{ relativeDir: string; - resultText: string; + sha?: string; + resultText?: string; failed: boolean; }> => { // Bit of a hack to make file resolutions happen from the correct directory @@ -89,6 +157,35 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix, extensions: ['js', 'jsx', 'ts', 'tsx', 'mjs', 'cjs'], }); + + let sha: string | undefined = undefined; + if (shouldCache) { + const result = await recursiveReadDir(fullDir); + + const hash = crypto.createHash('sha1'); + hash.update(parentHash!); + hash.update('\0'); + + for (const path of result.sort()) { + if (await eslint.isPathIgnored(path)) { + continue; + } + hash.update(workerRelativePath(fullDir, path)); + hash.update('\0'); + hash.update(await readFile(path)); + hash.update('\0'); + hash.update( + JSON.stringify(await eslint.calculateConfigForFile(path)), + ); + hash.update('\0'); + } + sha = await hash.digest('hex'); + if (successCache?.includes(sha)) { + console.log(`Skipped ${relativeDir} due to cache hit`); + return { relativeDir, sha, failed: false }; + } + } + const formatter = await eslint.loadFormatter(format); const results = await eslint.lintFiles(['.']); @@ -112,13 +209,21 @@ export async function command(opts: OptionValues, cmd: Command): Promise { relativeDir, resultText, failed, + sha, }; }; }, }); + const outputSuccessCache = []; + let failed = false; - for (const { relativeDir, resultText, failed: runFailed } of resultsList) { + for (const { + relativeDir, + resultText, + failed: runFailed, + sha, + } of resultsList) { if (runFailed) { console.log(chalk.red(`Lint failed in ${relativeDir}`)); failed = true; @@ -129,9 +234,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { console.log(); console.log(resultText.trimStart()); } + } else if (sha) { + outputSuccessCache.push(sha); } } + if (cacheDir) { + await writeCache(cacheDir, outputSuccessCache); + } + if (failed) { process.exit(1); }