paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #29573 from aramissennyeydd/sennyeya/api-reference-build

Browse Source 
chore: start building new api reference docs alongside existing build
This commit is contained in:
Aramis Sennyey
2025-05-23 11:31:33 -04:00
committed by GitHub
parent 148e7c8808 30474c43ae
commit 2a0290a2a8
11 changed files with 565 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/salty-worlds-float.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/create-app': minor
---
Add .cache directory to shipped gitignore.
.changeset/six-mangos-yell.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/repo-tools': minor
---
Add support for caching the per-package output from the `package-docs` command.
.github/workflows/deploy_microsite.yml
+67 -1
View File
@@ -80,6 +80,34 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: stable-reference-beta
path: type-docs/
if-no-files-found: error
retention-days: 1
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
- name: microsite yarn install
run: yarn install --immutable
working-directory: microsite
@@ -136,6 +164,34 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: next-reference-beta
path: type-docs/
if-no-files-found: error
retention-days: 1
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }}
# Also build and upload storybook
- name: storybook yarn install
run: yarn install --immutable
@@ -265,8 +321,18 @@ jobs:
name: storybook
path: microsite/build/storybook
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
with:
name: stable-reference-beta
path: microsite/build/api/stable/
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
with:
name: next-reference-beta
path: microsite/build/api/next/
- name: Check the build output
run: ls microsite/build && ls microsite/build/storybook
run: ls microsite/build && ls microsite/build/storybook && ls microsite/build/api/stable && ls microsite/build/api/next
- name: Deploy both microsite and storybook to gh-pages
uses: JamesIves/github-pages-deploy-action@6c2d9db40f9296374acc17b90404b6e8864128c8 # v4.7.3
.github/workflows/verify_microsite.yml
+64
View File
@@ -85,6 +85,32 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: stable-reference-beta
path: type-docs/
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
- name: microsite yarn install
run: yarn install --immutable
working-directory: microsite
@@ -140,6 +166,32 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: next-reference-beta
path: type-docs/
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }}
# Also build and upload storybook
- name: storybook yarn install
run: yarn install --immutable
@@ -271,3 +323,15 @@ jobs:
- name: build microsite
run: yarn build
working-directory: microsite
- name: download stable reference (beta)
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
with:
name: stable-reference-beta
path: api/stable/
- name: download next reference (beta)
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
with:
name: next-reference-beta
path: api/next/
packages/create-app/templates/default-app/.gitignore.hbs
+3
View File
@@ -51,3 +51,6 @@ site
# E2E test reports
e2e-test-report/
# Cache
.cache/
packages/repo-tools/package.json
+3 -1
View File
@@ -69,6 +69,7 @@
"commander": "^12.0.0",
"fs-extra": "^11.2.0",
"glob": "^8.0.3",
"globby": "^11.0.0",
"is-glob": "^4.0.3",
"js-yaml": "^4.1.0",
"just-diff": "^6.0.2",
@@ -81,7 +82,8 @@
"portfinder": "^1.0.32",
"tar": "^6.1.12",
"ts-morph": "^24.0.0",
"yaml-diff-patch": "^2.0.0"
"yaml-diff-patch": "^2.0.0",
"zod": "^3.22.4"
},
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
packages/repo-tools/src/commands/package-docs/Cache.test.ts
+178
View File
@@ -0,0 +1,178 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Lockfile } from '@backstage/cli-node';
import { PackageDocsCache } from './Cache';
import {
createMockDirectory,
MockDirectory,
} from '@backstage/backend-test-utils';
import { readFile } from 'fs/promises';
import { join as joinPath } from 'path';
jest.mock('crypto', () => {
const hash = {
update: jest.fn(),
digest: jest.fn().mockReturnValue('test'),
};
return {
createHash: jest.fn().mockReturnValue(hash),
};
});
describe('PackageDocsCache', () => {
let testDir: MockDirectory;
beforeAll(async () => {
testDir = createMockDirectory();
});
afterEach(async () => {
testDir.clear();
});
it('should be able to parse cache files', async () => {
testDir.addContent({
'.cache': {
'package-docs': {
test: {
'cache.json': JSON.stringify({
hash: 'test',
packageName: 'test',
restoreTo: 'test',
version: '1',
}),
},
},
},
test: {
'package.json': JSON.stringify({
name: '@test/test',
}),
},
});
const lockfile = {
getDependencyTreeHash: () => 'test',
} as any as Lockfile;
const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile);
expect(await cache.has('test')).toBe(true);
});
it('should be able to restore cache', async () => {
testDir.addContent({
'.cache': {
'package-docs': {
test: {
'cache.json': JSON.stringify({
hash: 'test',
packageName: 'test',
restoreTo: 'test',
version: '1',
}),
contents: {
'src/index.ts': 'export const test = "test";',
},
},
},
},
test: {
'package.json': JSON.stringify({
name: '@test/test',
}),
},
});
const lockfile = {
getDependencyTreeHash: () => 'test',
} as any as Lockfile;
const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile);
await cache.restore('test');
expect(
await readFile(joinPath(testDir.path, 'test', 'src/index.ts'), 'utf-8'),
).toBe('export const test = "test";');
});
it('should be able to write cache', async () => {
testDir.addContent({
'.cache': {},
test: {
'package.json': JSON.stringify({
name: '@test/test',
}),
'src/index.ts': 'export const test = "test";',
},
});
const lockfile = {
getDependencyTreeHash: () => 'test',
} as any as Lockfile;
const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile);
await cache.write('test', joinPath(testDir.path, 'test'));
expect(
await readFile(
joinPath(testDir.path, '.cache', 'package-docs', 'test', 'cache.json'),
'utf-8',
),
).toBe(
JSON.stringify({
hash: 'test',
packageName: '@test/test',
restoreTo: 'test',
version: '1',
}),
);
expect(
await readFile(
joinPath(
testDir.path,
'.cache',
'package-docs',
'test',
'contents',
'src/index.ts',
),
'utf-8',
),
).toBe('export const test = "test";');
});
it.each([
{
content: JSON.stringify({
hash: 'test',
packageName: 'test',
restoreTo: 'test',
version: '2',
}),
},
{
content: JSON.stringify({
hash: 'test',
packageName: 1,
}),
},
])('should skip invalid cache files', async content => {
testDir.addContent({
'.cache': {},
test: {
'package.json': JSON.stringify({
name: '@test/test',
}),
},
'cache.json': content,
});
const lockfile = {
getDependencyTreeHash: () => 'test',
} as any as Lockfile;
const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile);
expect(await cache.has('test')).toBe(false);
});
});
packages/repo-tools/src/commands/package-docs/Cache.ts
+158
View File
@@ -0,0 +1,158 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { readFile, writeFile, cp } from 'fs/promises';
import globby from 'globby';
import { dirname, join as joinPath, relative } from 'path';
import crypto from 'crypto';
import { Lockfile } from '@backstage/cli-node';
import { exists, rm, mkdirp } from 'fs-extra';
import { z } from 'zod';
import { CACHE_DIR, CACHE_FILE } from './constants';
const version = '1';
interface CacheEntry {
hash: string;
packageName: string;
restoreTo: string;
version: string;
}
const cacheEntrySchema = z.object({
hash: z.string(),
packageName: z.string(),
restoreTo: z.string(),
version: z.string(),
});
export class PackageDocsCache {
// A map of package directory to package hash.
private keyCache: Map<string, string>;
constructor(
private readonly lockfile: Lockfile,
// A map of package directory to cache entry.
private readonly cache: Map<string, CacheEntry>,
private readonly baseDirectory: string,
) {
this.keyCache = new Map();
}
static async loadAsync(baseDirectory: string, lockfile: Lockfile) {
const cacheDir = joinPath(baseDirectory, CACHE_DIR);
await mkdirp(cacheDir);
const cacheFiles = await globby(`**/${CACHE_FILE}`, {
cwd: cacheDir,
});
const map = new Map<string, CacheEntry>();
for (const file of cacheFiles) {
const pkg = dirname(file);
const cache = await readFile(joinPath(cacheDir, file), 'utf-8');
try {
const cacheJson = JSON.parse(cache);
const parsed = cacheEntrySchema.parse(cacheJson);
if (parsed.version !== version) {
console.warn(
`Skipping cache file ${file} due to version mismatch: ${parsed.version} !== ${version}`,
);
continue;
}
map.set(pkg, parsed);
} catch (e) {
console.error(`Skipping unparseable cache file ${file}: ${e}`);
}
}
return new PackageDocsCache(lockfile, map, baseDirectory);
}
async directoryToName(directory: string) {
const packageJson = await readFile(
joinPath(this.baseDirectory, directory, 'package.json'),
'utf-8',
);
return JSON.parse(packageJson).name;
}
private async toKey(pkg: string) {
if (this.keyCache.has(pkg)) {
return this.keyCache.get(pkg)!;
}
const name = await this.directoryToName(pkg);
const result = await globby('src/**', {
gitignore: true,
onlyFiles: true,
cwd: pkg,
});
const hash = crypto.createHash('sha1');
hash.update(version);
hash.update('\0');
for (const path of result.sort()) {
const absPath = joinPath(this.baseDirectory, pkg, path);
const pathInPackage = joinPath(absPath, path);
hash.update(pathInPackage);
hash.update('\0');
hash.update(await readFile(absPath));
hash.update('\0');
}
hash.update(this.lockfile.getDependencyTreeHash(name));
hash.update('\0');
const hashString = hash.digest('hex');
this.keyCache.set(pkg, hashString);
return hashString;
}
async has(pkg: string) {
const cache = this.cache.get(pkg);
if (!cache) {
return false;
}
const hashString = await this.toKey(pkg);
return cache.hash === hashString;
}
async restore(pkg: string) {
if (!this.has(pkg)) {
throw new Error(`Cache entry for ${pkg} not found`);
}
const cacheEntry = this.cache.get(pkg);
const restoreTo = cacheEntry!.restoreTo;
const cacheDir = joinPath(this.baseDirectory, CACHE_DIR, pkg);
const contentsDir = joinPath(cacheDir, 'contents');
const targetDir = joinPath(this.baseDirectory, restoreTo);
await mkdirp(targetDir);
await cp(contentsDir, targetDir, { recursive: true });
}
async write(pkg: string, contentDirectory: string) {
const cacheDir = joinPath(this.baseDirectory, CACHE_DIR, pkg);
const contentsDir = joinPath(cacheDir, 'contents');
if (await exists(contentsDir)) {
await rm(contentsDir, { recursive: true });
} else {
await mkdirp(contentsDir);
}
const hashString = await this.toKey(pkg);
await cp(contentDirectory, contentsDir, { recursive: true });
const cacheEntry: CacheEntry = {
hash: hashString,
packageName: await this.directoryToName(pkg),
restoreTo: relative(this.baseDirectory, contentDirectory),
version,
};
await writeFile(joinPath(cacheDir, CACHE_FILE), JSON.stringify(cacheEntry));
}
}
packages/repo-tools/src/commands/package-docs/command.ts
+59 -35
View File
@@ -17,8 +17,11 @@ import { exec } from 'child_process';
import { promisify } from 'util';
import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
import { createTemporaryTsConfig } from './utils';
import { mkdir, readFile, writeFile } from 'fs/promises';
import { readFile, writeFile } from 'fs/promises';
import pLimit from 'p-limit';
import { mkdirp } from 'fs-extra';
import { PackageDocsCache } from './Cache';
import { Lockfile } from '@backstage/cli-node';
const limit = pLimit(8);
@@ -38,6 +41,7 @@ const EXCLUDE = [
'packages/techdocs-cli-embedded-app',
'packages/yarn-plugin',
'packages/backend',
'packages/backend-legacy',
];
const HIGHLIGHT_LANGUAGES = [
@@ -75,41 +79,35 @@ async function generateDocJson(pkg: string) {
!exports.length ||
!exports.some(e => e.startsWith('src') || e.startsWith('./src'))
) {
return;
return false;
}
try {
await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), {
recursive: true,
});
await mkdirp(cliPaths.resolveTargetRoot(`dist-types`, pkg));
const { stdout, stderr } = await execAsync(
[
cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
'--json',
cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'),
'--tsconfig',
temporaryTsConfigPath,
'--basePath',
cliPaths.targetRoot,
'--skipErrorChecking',
...(getExports(packageJson).flatMap(e => [
'--entryPoints',
e,
]) as string[]),
].join(' '),
{
cwd: pkg,
env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
},
);
console.log(`### Processed ${pkg}`);
console.log(stdout);
console.error(stderr);
} catch (e) {
console.error('Failed to generate docs for', pkg);
console.error(e);
}
const { stdout, stderr } = await execAsync(
[
cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
'--json',
cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'),
'--tsconfig',
temporaryTsConfigPath,
'--basePath',
cliPaths.targetRoot,
'--skipErrorChecking',
...(getExports(packageJson).flatMap(e => [
'--entryPoints',
e,
]) as string[]),
].join(' '),
{
cwd: pkg,
env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
},
);
console.log(`### Processed ${pkg}`);
console.log(stdout);
console.error(stderr);
return true;
}
export default async function packageDocs(paths: string[] = [], opts: any) {
@@ -120,6 +118,11 @@ export default async function packageDocs(paths: string[] = [], opts: any) {
exclude: opts.exclude,
});
const cache = await PackageDocsCache.loadAsync(
cliPaths.resolveTargetRoot(),
await Lockfile.load(cliPaths.resolveTargetRoot('yarn.lock')),
);
console.log(`### Generating docs.`);
await Promise.all(
selectedPackageDirs.map(pkg =>
@@ -127,8 +130,29 @@ export default async function packageDocs(paths: string[] = [], opts: any) {
if (EXCLUDE.includes(pkg)) {
return;
}
console.log(`### Processing ${pkg}`);
await generateDocJson(pkg);
if (await cache.has(pkg)) {
console.log(`### Skipping ${pkg} due to cache hit`);
try {
await cache.restore(pkg);
return;
} catch (e) {
console.error('Failed to restore cache for', pkg);
console.error(e);
}
}
try {
console.log(`### Processing ${pkg}`);
const success = await generateDocJson(pkg);
if (success) {
await cache.write(
pkg,
cliPaths.resolveTargetRoot(`dist-types`, pkg),
);
}
} catch (e) {
console.error('Failed to generate docs for', pkg);
console.error(e);
}
}),
),
);
packages/repo-tools/src/commands/package-docs/constants.ts
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const CACHE_FILE = 'cache.json';
export const CACHE_DIR = '.cache/package-docs';
yarn.lock
+5 -3
View File
@@ -8473,6 +8473,7 @@ __metadata:
commander: "npm:^12.0.0"
fs-extra: "npm:^11.2.0"
glob: "npm:^8.0.3"
globby: "npm:^11.0.0"
is-glob: "npm:^4.0.3"
js-yaml: "npm:^4.1.0"
just-diff: "npm:^6.0.2"
@@ -8487,6 +8488,7 @@ __metadata:
ts-morph: "npm:^24.0.0"
typedoc: "npm:^0.28.0"
yaml-diff-patch: "npm:^2.0.0"
zod: "npm:^3.22.4"
peerDependencies:
"@microsoft/api-extractor-model": "*"
"@microsoft/tsdoc": "*"
@@ -48920,9 +48922,9 @@ __metadata:
linkType: hard
"zod@npm:^3.22.4":
version: 3.23.8
resolution: "zod@npm:3.23.8"
checksum: 10/846fd73e1af0def79c19d510ea9e4a795544a67d5b34b7e1c4d0425bf6bfd1c719446d94cdfa1721c1987d891321d61f779e8236fde517dc0e524aa851a6eff1
version: 3.24.4
resolution: "zod@npm:3.24.4"
checksum: 10/3d545792fa54bb27ee5dbc34a5709e81f603185fcc94c8204b5d95c20dc4c81d870ff9c51f3884a30ef05cdc601449f4c4df254ac4783f0827b1faed7c1cdb48
languageName: node
linkType: hard