diff --git a/.changeset/techdocs-loud-spies-attack.md b/.changeset/techdocs-loud-spies-attack.md new file mode 100644 index 0000000000..ea80c73116 --- /dev/null +++ b/.changeset/techdocs-loud-spies-attack.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-techdocs': minor +'@backstage/plugin-techdocs-backend': minor +--- + +Rewrite the `/sync/:namespace/:kind/:name` to return an event-stream. +This change allows the sync process to take longer than a normal HTTP timeout. +The stream also emits log events, so the caller can follow the build process in the frontend. diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 0f0752a0c2..9fb6671031 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.8.4", + "@backstage/catalog-client": "^0.3.15", "@backstage/catalog-model": "^0.8.4", "@backstage/config": "^0.1.5", "@backstage/errors": "^0.1.1", @@ -47,6 +48,7 @@ "devDependencies": { "@backstage/cli": "^0.7.3", "@types/dockerode": "^3.2.1", + "msw": "^0.29.0", "supertest": "^6.1.3" }, "files": [ diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index e9df0b8400..a66125521d 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -18,6 +18,7 @@ import { ENTITY_DEFAULT_NAMESPACE, stringifyEntityRef, } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import { NotModifiedError } from '@backstage/errors'; import { GeneratorBase, @@ -31,8 +32,8 @@ import { import fs from 'fs-extra'; import os from 'os'; import path from 'path'; +import { Writable } from 'stream'; import { Logger } from 'winston'; -import { Config } from '@backstage/config'; import { BuildMetadataStorage } from './BuildMetadataStorage'; type DocsBuilderArguments = { @@ -42,6 +43,7 @@ type DocsBuilderArguments = { entity: Entity; logger: Logger; config: Config; + logStream?: Writable; }; export class DocsBuilder { @@ -51,6 +53,7 @@ export class DocsBuilder { private entity: Entity; private logger: Logger; private config: Config; + private logStream: Writable | undefined; constructor({ preparers, @@ -59,6 +62,7 @@ export class DocsBuilder { entity, logger, config, + logStream, }: DocsBuilderArguments) { this.preparer = preparers.get(entity); this.generator = generators.get(entity); @@ -66,6 +70,7 @@ export class DocsBuilder { this.entity = entity; this.logger = logger; this.config = config; + this.logStream = logStream; } /** @@ -159,12 +164,14 @@ export class DocsBuilder { const outputDir = await fs.mkdtemp( path.join(tmpdirResolvedPath, 'techdocs-tmp-'), ); + const parsedLocationAnnotation = getLocationForEntity(this.entity); await this.generator.run({ inputDir: preparedDir, outputDir, parsedLocationAnnotation, etag: newEtag, + logStream: this.logStream, }); // Remove Prepared directory since it is no longer needed. diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts new file mode 100644 index 0000000000..a3e17589ae --- /dev/null +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -0,0 +1,267 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + errorHandler, + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { + GeneratorBuilder, + PreparerBuilder, + PublisherBase, +} from '@backstage/techdocs-common'; +import express from 'express'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import request from 'supertest'; +import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; +import { createRouter } from './router'; + +jest.mock('@backstage/config'); +jest.mock('../DocsBuilder'); + +const MockedConfigReader = ConfigReader as jest.MockedClass< + typeof ConfigReader +>; +const MockedDocsBuilder = DocsBuilder as jest.MockedClass; + +const server = setupServer(); + +describe('createRouter', () => { + // the calls from supertest should not be handled by msw so we only warn onUnhandledRequest + beforeAll(() => server.listen({ onUnhandledRequest: 'warn' })); + afterAll(() => server.close()); + afterEach(() => server.resetHandlers()); + + const preparers: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const generators: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const publisher: jest.Mocked = { + docsRouter: jest.fn(), + fetchTechDocsMetadata: jest.fn(), + getReadiness: jest.fn(), + hasDocsBeenGenerated: jest.fn(), + publish: jest.fn(), + }; + const discovery: jest.Mocked = { + getBaseUrl: jest.fn(), + getExternalBaseUrl: jest.fn(), + }; + + let app: express.Express; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + beforeEach(async () => { + publisher.docsRouter.mockReturnValue(() => {}); + discovery.getBaseUrl.mockImplementation(async type => { + return `http://backstage.local/api/${type}`; + }); + + const router = await createRouter({ + preparers, + generators, + publisher, + config: new ConfigReader({}), + logger: getVoidLogger(), + discovery, + }); + + router.use(errorHandler()); + app = express(); + app.use(router); + }); + + describe('GET /sync/:namespace/:kind/:name', () => { + it('should execute an update', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + MockedConfigReader.prototype.getString.mockReturnValue('local'); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + annotations: { + 'sda.se/release-notes-location': + 'github-releases:https://github.com/backstage/backstage', + }, + }, + }; + + server.use( + rest.get( + 'http://backstage.local/api/catalog/entities/by-name/Component/default/test', + (_req, res, ctx) => { + return res(ctx.json(entity)); + }, + ), + ); + + MockedDocsBuilder.prototype.build.mockImplementation(async () => { + // extract the logStream from the constructor call + const logStream = MockedDocsBuilder.mock.calls[0][0].logStream; + + logStream?.write('Some log'); + logStream?.write('Another log'); + + return true; + }); + + publisher.hasDocsBeenGenerated.mockResolvedValue(true); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: log +data: "Some log" + +event: log +data: "Another log" + +event: finish +data: {"updated":true} + +`, + ); + + expect(shouldCheckForUpdate).toBeCalledTimes(1); + expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1); + expect(DocsBuilder.prototype.build).toBeCalledTimes(1); + }); + + it('should not check for an update too often', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(false); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + annotations: { + 'sda.se/release-notes-location': + 'github-releases:https://github.com/backstage/backstage', + }, + }, + }; + + server.use( + rest.get( + 'http://backstage.local/api/catalog/entities/by-name/Component/default/test', + (_req, res, ctx) => { + return res(ctx.json(entity)); + }, + ), + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: finish +data: {"updated":false} + +`, + ); + + expect(shouldCheckForUpdate).toBeCalledTimes(1); + expect(MockedConfigReader.prototype.getString).toBeCalledTimes(0); + expect(DocsBuilder.prototype.build).toBeCalledTimes(0); + }); + + it('should not check for an update without local builder', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + MockedConfigReader.prototype.getString.mockReturnValue('external'); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + annotations: { + 'sda.se/release-notes-location': + 'github-releases:https://github.com/backstage/backstage', + }, + }, + }; + + server.use( + rest.get( + 'http://backstage.local/api/catalog/entities/by-name/Component/default/test', + (_req, res, ctx) => { + return res(ctx.json(entity)); + }, + ), + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: finish +data: {"updated":false} + +`, + ); + + expect(shouldCheckForUpdate).toBeCalledTimes(1); + expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1); + expect(DocsBuilder.prototype.build).toBeCalledTimes(0); + }); + + it('rejects when entity is not found', async () => { + server.use( + rest.get( + 'http://backstage.local/api/catalog/entities/by-name/Component/default/test', + (_req, res, ctx) => { + return res(ctx.status(404)); + }, + ), + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(404); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index eceae2e3d3..08c5301f20 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -14,9 +14,10 @@ * limitations under the License. */ import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { CatalogClient } from '@backstage/catalog-client'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { NotFoundError } from '@backstage/errors'; import { GeneratorBuilder, getLocationForEntity, @@ -24,12 +25,12 @@ import { PublisherBase, } from '@backstage/techdocs-common'; import fetch from 'cross-fetch'; -import express from 'express'; +import express, { Response } from 'express'; import Router from 'express-promise-router'; import { Knex } from 'knex'; +import { PassThrough } from 'stream'; import { Logger } from 'winston'; -import { DocsBuilder } from '../DocsBuilder'; -import { shouldCheckForUpdate } from '../DocsBuilder/BuildMetadataStorage'; +import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; type RouterOptions = { preparers: PreparerBuilder; @@ -50,6 +51,7 @@ export async function createRouter({ discovery, }: RouterOptions): Promise { const router = Router(); + const catalogClient = new CatalogClient({ discoveryApi: discovery }); router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; @@ -108,56 +110,45 @@ export async function createRouter({ }); // Check if docs are the latest version and trigger rebuilds if not - // Responds with immediate success if rebuild not needed + // Responds with an event-stream that closes after the build finished + // Responds with an immediate success if rebuild not needed // If a build is required, responds with a success when finished router.get('/sync/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; - const catalogUrl = await discovery.getBaseUrl('catalog'); - const triple = [kind, namespace, name].map(encodeURIComponent).join('/'); - const token = getBearerToken(req.headers.authorization); - const catalogRes = await fetch(`${catalogUrl}/entities/by-name/${triple}`, { - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!catalogRes.ok) { - const catalogResText = await catalogRes.text(); - res.status(catalogRes.status); - res.send(catalogResText); - return; - } - const entity: Entity = await catalogRes.json(); + const entity = await catalogClient.getEntityByName( + { kind, namespace, name }, + { token }, + ); - if (!entity.metadata.uid) { + if (!entity?.metadata?.uid) { throw new NotFoundError('Entity metadata UID missing'); } + + // open the event-stream + const { log, error, finish } = createEventStream(res); + + // create an in-memory stream to forward logs to the event-stream + const logStream = new PassThrough(); + logStream.on('data', async data => { + log(data.toString().trim()); + }); + + // check if the last update check was too recent if (!shouldCheckForUpdate(entity.metadata.uid)) { - res.status(200).json({ - message: `Last check for documentation update is recent, did not retry.`, - }); + finish({ updated: false }); return; } - let publisherType = ''; - try { - publisherType = config.getString('techdocs.publisher.type'); - } catch (err) { - throw new Error( - 'Unable to get techdocs.publisher.type in your app config. Set it to either ' + - "'local', 'googleGcs' or other support storage providers. Read more here " + - 'https://backstage.io/docs/features/techdocs/architecture', - ); - } // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline // of the repository) is responsible for building and publishing documentation to the storage provider if (config.getString('techdocs.builder') !== 'local') { - res.status(200).json({ - message: - '`techdocs.builder` app config is not set to `local`, so docs will not be generated locally and sync is not required.', - }); + finish({ updated: false }); return; } + const docsBuilder = new DocsBuilder({ preparers, generators, @@ -165,52 +156,41 @@ export async function createRouter({ logger, entity, config, + logStream, }); + let foundDocs = false; - switch (publisherType) { - case 'local': - case 'awsS3': - case 'azureBlobStorage': - case 'openStackSwift': - case 'googleGcs': { - // This block should be valid for all storage implementations. So no need to duplicate in future, - // add the publisher type in the list here. - const updated = await docsBuilder.build(); - if (!updated) { - throw new NotModifiedError(); - } + const updated = await docsBuilder.build(); - // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched - // on the user's page. If not, respond with a message asking them to check back later. - // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second - for (let attempt = 0; attempt < 5; attempt++) { - if (await publisher.hasDocsBeenGenerated(entity)) { - foundDocs = true; - break; - } - await new Promise(r => setTimeout(r, 1000)); - } - if (!foundDocs) { - logger.error( - 'Published files are taking longer to show up in storage. Something went wrong.', - ); - throw new NotFoundError( - 'Sorry! It took too long for the generated docs to show up in storage. Check back later.', - ); - } + if (!updated) { + finish({ updated: false }); + return; + } - res - .status(201) - .json({ message: 'Docs updated or did not need updating' }); + // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched + // on the user's page. If not, respond with a message asking them to check back later. + // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second + for (let attempt = 0; attempt < 5; attempt++) { + if (await publisher.hasDocsBeenGenerated(entity)) { + foundDocs = true; break; } - - default: - throw new NotFoundError( - `Publisher type ${publisherType} is not supported by techdocs-backend docs builder.`, - ); + await new Promise(r => setTimeout(r, 1000)); } + if (!foundDocs) { + logger.error( + 'Published files are taking longer to show up in storage. Something went wrong.', + ); + error( + new NotFoundError( + 'Sorry! It took too long for the generated docs to show up in storage. Check back later.', + ), + ); + return; + } + + finish({ updated: true }); }); // Route middleware which serves files from the storage set in the publisher. @@ -222,3 +202,56 @@ export async function createRouter({ function getBearerToken(header?: string): string | undefined { return header?.match(/(?:Bearer)\s+(\S+)/i)?.[1]; } + +/** + * Create an event-stream response that emits the events 'log', 'error', and 'finish'. + * + * @param res the response to write the event-stream to + * @returns A tuple of callbacks to emit messages. A call to 'error' or 'finish' + * will close the event-stream. + */ +function createEventStream( + res: Response, +): { + log: (message: string) => void; + error: (e: Error) => void; + finish: (result: { updated: boolean }) => void; +} { + // Mandatory headers and http status to keep connection open + res.writeHead(200, { + Connection: 'keep-alive', + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }); + + // client closes connection + res.socket?.on('close', () => { + res.end(); + }); + + // write the event to the stream + const send = (type: 'error' | 'finish' | 'log', data: any) => { + res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); + + // res.flush() is only available with the compression middleware + if (res.flush) { + res.flush(); + } + }; + + return { + log: data => { + send('log', data); + }, + + error: e => { + send('error', e.message); + res.end(); + }, + + finish: result => { + send('finish', result); + res.end(); + }, + }; +} diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index dbd5e3ab26..825db437d9 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -44,7 +44,7 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element; export const Router: () => JSX.Element; // @public (undocumented) -export type SyncResult = 'cached' | 'updated' | 'timeout'; +export type SyncResult = 'cached' | 'updated'; // @public (undocumented) export interface TechDocsApi { @@ -112,7 +112,7 @@ export interface TechDocsStorageApi { // (undocumented) getStorageUrl(): Promise; // (undocumented) - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise; } // @public (undocumented) @@ -140,7 +140,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise; } diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index cdc81caa5a..80e81fe178 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -17,6 +17,7 @@ import { createDevApp } from '@backstage/dev-utils'; import { NotFoundError } from '@backstage/errors'; import React from 'react'; +import { EntityName } from '@backstage/catalog-model'; import { Reader, SyncResult, @@ -81,9 +82,18 @@ function createPage({ }); } - async syncEntityDocs() { + async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) { if (syncDocsDelay) { - await new Promise(resolve => setTimeout(resolve, syncDocsDelay)); + for (let i = 0; i < 10; i++) { + setTimeout( + () => logHandler?.call(this, `Log line ${i}`), + ((i + 1) * syncDocsDelay) / 10, + ); + } + + await new Promise(resolve => { + setTimeout(resolve, syncDocsDelay); + }); } return syncDocs(); @@ -195,13 +205,6 @@ createDevApp() syncDocsDelay: 2000, })} - - - {createPage({ - syncDocs: () => 'timeout', - syncDocsDelay: 2000, - })} - ), diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 1a5abc6fbf..1e67549e23 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -44,6 +44,7 @@ "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@material-ui/styles": "^4.10.0", + "eventsource": "^1.1.0", "react": "^16.13.1", "react-dom": "^16.13.1", "react-router": "6.0.0-beta.0", @@ -60,6 +61,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/react-hooks": "^3.4.2", "@testing-library/user-event": "^13.1.8", + "@types/eventsource": "^1.1.5", "@types/jest": "^26.0.7", "@types/node": "^14.14.32", "@types/react": "^16.9", diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 739c5fc133..a23d9fc7f0 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -28,14 +28,17 @@ export const techdocsApiRef = createApiRef({ description: 'Used to make requests towards techdocs API', }); -export type SyncResult = 'cached' | 'updated' | 'timeout'; +export type SyncResult = 'cached' | 'updated'; export interface TechDocsStorageApi { getApiOrigin(): Promise; getStorageUrl(): Promise; getBuilder(): Promise; getEntityDocs(entityId: EntityName, path: string): Promise; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs( + entityId: EntityName, + logHandler?: (line: string) => void, + ): Promise; getBaseUrl( oldBaseUrl: string, entityId: EntityName, diff --git a/plugins/techdocs/src/client.test.ts b/plugins/techdocs/src/client.test.ts index 329d3cbcb6..4cc7aca6a6 100644 --- a/plugins/techdocs/src/client.test.ts +++ b/plugins/techdocs/src/client.test.ts @@ -13,9 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Config } from '@backstage/config'; -import { TechDocsStorageClient } from './client'; import { UrlPatternDiscovery } from '@backstage/core-app-api'; +import { IdentityApi } from '@backstage/core-plugin-api'; +import { NotFoundError } from '@backstage/errors'; +import EventSource from 'eventsource'; +import { TechDocsStorageClient } from './client'; + +const MockedEventSource: jest.MockedClass< + typeof EventSource +> = EventSource as any; + +jest.mock('eventsource'); const mockEntity = { kind: 'Component', @@ -29,6 +39,16 @@ describe('TechDocsStorageClient', () => { getOptionalString: () => 'http://backstage:9191/api/techdocs', } as Partial; const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); + const identityApi: jest.Mocked = { + getIdToken: jest.fn(), + getProfile: jest.fn(), + getUserId: jest.fn(), + signOut: jest.fn(), + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); it('should return correct base url based on defined storage', async () => { // @ts-ignore Partial not assignable to Config. @@ -51,4 +71,183 @@ describe('TechDocsStorageClient', () => { `${mockBaseUrl}/static/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`, ); }); + + describe('syncEntityDocs', () => { + it('should create eventsource without headers', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + await storageApi.syncEntityDocs(mockEntity); + + expect( + MockedEventSource, + ).toBeCalledWith( + 'http://backstage:9191/api/techdocs/sync/default/Component/test-component', + { withCredentials: true, headers: {} }, + ); + }); + + it('should create eventsource with headers', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + identityApi.getIdToken.mockResolvedValue('token'); + + await storageApi.syncEntityDocs(mockEntity); + + expect( + MockedEventSource, + ).toBeCalledWith( + 'http://backstage:9191/api/techdocs/sync/default/Component/test-component', + { withCredentials: true, headers: { Authorization: 'Bearer token' } }, + ); + }); + + it('should resolve to cached', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual( + 'cached', + ); + }); + + it('should resolve to updated', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": true}' } as any); + } + }, + ); + + await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual( + 'updated', + ); + }); + + it('should log values', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'log') { + fn({ data: '"A log message"' } as any); + } + + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + const logHandler = jest.fn(); + await expect( + storageApi.syncEntityDocs(mockEntity, logHandler), + ).resolves.toEqual('cached'); + + expect(logHandler).toBeCalledTimes(1); + expect(logHandler).toBeCalledWith('A log message'); + }); + + it('should throw NotFoundError', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + // we await later after we emitted the error + const promise = storageApi.syncEntityDocs(mockEntity).then(); + + // flush the event loop + await new Promise(setImmediate); + + const instance = MockedEventSource.mock + .instances[0] as jest.Mocked; + + instance.onerror({ + status: 404, + message: 'Some not found warning', + } as any); + + await expect(promise).rejects.toThrow(NotFoundError); + await expect(promise).rejects.toThrowError('Some not found warning'); + }); + + it('should throw generic errors', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + // we await later after we emitted the error + const promise = storageApi.syncEntityDocs(mockEntity).then(); + + // flush the event loop + await new Promise(setImmediate); + + const instance = MockedEventSource.mock + .instances[0] as jest.Mocked; + + instance.onerror({ + status: 500, + message: 'Some other error', + } as any); + + await expect(promise).rejects.toThrow(Error); + await expect(promise).rejects.toThrowError('Some other error'); + }); + }); }); diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index 19fc565a2f..29e6e018d0 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -16,10 +16,11 @@ import { EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { NotFoundError } from '@backstage/errors'; +import EventSource from 'eventsource'; import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api'; import { TechDocsEntityMetadata, TechDocsMetadata } from './types'; -import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; /** * API to talk to techdocs-backend. @@ -192,44 +193,59 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * Check if docs are on the latest version and trigger rebuild if not * * @param {EntityName} entityId Object containing entity data like name, namespace, etc. + * @param {Function} logHandler Callback to receive log messages from the build process * @returns {SyncResult} Whether documents are currently synchronized to newest version * @throws {Error} Throws error on error from sync endpoint in Techdocs Backend */ - async syncEntityDocs(entityId: EntityName): Promise { + async syncEntityDocs( + entityId: EntityName, + logHandler: (line: string) => void = () => {}, + ): Promise { const { kind, namespace, name } = entityId; const apiOrigin = await this.getApiOrigin(); const url = `${apiOrigin}/sync/${namespace}/${kind}/${name}`; const token = await this.identityApi.getIdToken(); - let request; - let attempts: number = 0; - // retry if request times out, up to 5 times - // can happen due to docs taking too long to generate - while (!request || (request.status === 408 && attempts < 5)) { - attempts++; - request = await fetch(url, { + + return new Promise((resolve, reject) => { + const source = new EventSource(url, { + withCredentials: true, headers: token ? { Authorization: `Bearer ${token}` } : {}, }); - } - switch (request.status) { - case 404: - throw new NotFoundError((await request.json()).error); + source.addEventListener('log', (e: any) => { + if (e.data) { + logHandler(JSON.parse(e.data)); + } + }); - case 200: - case 304: - return 'cached'; + source.addEventListener('finish', (e: any) => { + let updated: boolean = false; - case 201: - return 'updated'; + if (e.data) { + ({ updated } = JSON.parse(e.data)); + } - // for timeout and misc errors, handle without error to allow viewing older docs - // if older docs not available, - // Reader will show 404 error coming from getEntityDocs - case 408: - default: - return 'timeout'; - } + resolve(updated ? 'updated' : 'cached'); + }); + + source.onerror = (e: any) => { + source.close(); + + switch (e.status) { + // the endpoint returned a 404 status + case 404: + reject(new NotFoundError(e.message)); + return; + + // also handles the event-stream close. the reject is ignored if the Promise was already + // resolved by a finish event. + default: + reject(new Error(e.message)); + return; + } + }; + }); } async getBaseUrl( diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index f80cf6cc4c..cc76d789b6 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -15,6 +15,7 @@ */ import { EntityName } from '@backstage/catalog-model'; +import { useApi } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; import { useTheme } from '@material-ui/core'; @@ -37,7 +38,6 @@ import { import { TechDocsNotFound } from './TechDocsNotFound'; import TechDocsProgressBar from './TechDocsProgressBar'; import { useReaderState } from './useReaderState'; -import { useApi } from '@backstage/core-plugin-api'; type Props = { entityId: EntityName; @@ -328,12 +328,6 @@ export const Reader = ({ entityId, onReady }: Props) => { to view. )} - {state === 'CONTENT_STALE_TIMEOUT' && ( - - Building a newer version of this documentation took longer than - expected. Please refresh to try again. - - )} {state === 'CONTENT_STALE_ERROR' && ( Building a newer version of this documentation failed. {errorMessage} diff --git a/plugins/techdocs/src/reader/components/useReaderState.test.tsx b/plugins/techdocs/src/reader/components/useReaderState.test.tsx index 1edf41af30..d3f5362a07 100644 --- a/plugins/techdocs/src/reader/components/useReaderState.test.tsx +++ b/plugins/techdocs/src/reader/components/useReaderState.test.tsx @@ -14,6 +14,7 @@ * limitations under the License. */ +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { NotFoundError } from '@backstage/errors'; import { act, renderHook } from '@testing-library/react-hooks'; import React from 'react'; @@ -23,7 +24,6 @@ import { reducer, useReaderState, } from './useReaderState'; -import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; describe('useReaderState', () => { let Wrapper: React.ComponentType; @@ -55,14 +55,12 @@ describe('useReaderState', () => { ${false} | ${undefined} | ${'BUILDING'} | ${'INITIAL_BUILD'} ${false} | ${undefined} | ${'BUILD_READY'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'} - ${false} | ${undefined} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'UP_TO_DATE'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'ERROR'} | ${'CONTENT_NOT_FOUND'} ${false} | ${'asdf'} | ${'CHECKING'} | ${'CONTENT_FRESH'} ${false} | ${'asdf'} | ${'BUILDING'} | ${'CONTENT_STALE_REFRESHING'} ${false} | ${'asdf'} | ${'BUILD_READY'} | ${'CONTENT_STALE_READY'} ${false} | ${'asdf'} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'} - ${false} | ${'asdf'} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_STALE_TIMEOUT'} ${false} | ${'asdf'} | ${'UP_TO_DATE'} | ${'CONTENT_FRESH'} ${false} | ${'asdf'} | ${'ERROR'} | ${'CONTENT_STALE_ERROR'} `( @@ -369,42 +367,6 @@ describe('useReaderState', () => { }); }); - it('should handle timed-out refresh', async () => { - techdocsStorageApi.getEntityDocs.mockResolvedValue('my content'); - techdocsStorageApi.syncEntityDocs.mockResolvedValue('timeout'); - - await act(async () => { - const { result, waitForValueToChange } = await renderHook( - () => useReaderState('Component', 'default', 'backstage', '/example'), - { wrapper: Wrapper }, - ); - - expect(result.current).toEqual({ - state: 'CHECKING', - content: undefined, - errorMessage: '', - }); - - // the content is returned but the sync is in progress - await waitForValueToChange(() => result.current.state); - expect(result.current).toEqual({ - state: 'CONTENT_STALE_TIMEOUT', - content: 'my content', - errorMessage: '', - }); - - expect(techdocsStorageApi.getEntityDocs).toBeCalledWith( - { kind: 'Component', namespace: 'default', name: 'backstage' }, - '/example', - ); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); - }); - }); - it('should handle content error', async () => { techdocsStorageApi.getEntityDocs.mockRejectedValue( new NotFoundError('Some error description'), diff --git a/plugins/techdocs/src/reader/components/useReaderState.ts b/plugins/techdocs/src/reader/components/useReaderState.ts index ac5145ee49..be47ee301d 100644 --- a/plugins/techdocs/src/reader/components/useReaderState.ts +++ b/plugins/techdocs/src/reader/components/useReaderState.ts @@ -14,10 +14,10 @@ * limitations under the License. */ +import { useApi } from '@backstage/core-plugin-api'; import { useEffect, useMemo, useReducer, useRef } from 'react'; import { useAsync, useAsyncRetry } from 'react-use'; import { techdocsStorageApiRef } from '../../api'; -import { useApi } from '@backstage/core-plugin-api'; /** * A state representation that is used to configure the UI of @@ -35,9 +35,6 @@ type ContentStateTypes = /** There is content, but after a reload, the content will be different */ | 'CONTENT_STALE_READY' - /** There is content, the backend tried to update it, but it took too long */ - | 'CONTENT_STALE_TIMEOUT' - /** There is content, the backend tried to update it, but failed */ | 'CONTENT_STALE_ERROR' @@ -93,11 +90,6 @@ export function calculateDisplayState({ return 'CONTENT_STALE_READY'; } - // the build timed out, but the content is still stale - if (activeSyncState === 'BUILD_TIMED_OUT') { - return 'CONTENT_STALE_TIMEOUT'; - } - // the build failed, but the content is still stale if (activeSyncState === 'ERROR') { return 'CONTENT_STALE_ERROR'; @@ -127,9 +119,6 @@ type SyncStates = */ | 'BUILD_READY_RELOAD' - /** Building the documentation timed out */ - | 'BUILD_TIMED_OUT' - /** No need for a sync. The content was already up-to-date. */ | 'UP_TO_DATE' @@ -274,18 +263,27 @@ export function useReaderState( name, }); - if (result === 'updated') { - // if there was no content prior to building, retry the loading - if (!contentRef.current.content) { - contentRef.current.reload(); - dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' }); - } else { - dispatch({ type: 'sync', state: 'BUILD_READY' }); - } - } else if (result === 'cached') { - dispatch({ type: 'sync', state: 'UP_TO_DATE' }); - } else { - dispatch({ type: 'sync', state: 'BUILD_TIMED_OUT' }); + switch (result) { + case 'updated': + // if there was no content prior to building, retry the loading + if (!contentRef.current.content) { + contentRef.current.reload(); + dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' }); + } else { + dispatch({ type: 'sync', state: 'BUILD_READY' }); + } + break; + case 'cached': + dispatch({ type: 'sync', state: 'UP_TO_DATE' }); + break; + + default: + dispatch({ + type: 'sync', + state: 'ERROR', + syncError: new Error('Unexpected return state'), + }); + break; } } catch (e) { dispatch({ type: 'sync', state: 'ERROR', syncError: e }); diff --git a/yarn.lock b/yarn.lock index b04756527c..4b0383a653 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5745,6 +5745,11 @@ resolved "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== +"@types/eventsource@^1.1.5": + version "1.1.5" + resolved "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.5.tgz#408e9b45efb176c8bea672ab58c81e7ab00d24bc" + integrity sha512-BA9q9uC2PAMkUS7DunHTxWZZaVpeNzDG8lkBxcKwzKJClfDQ4Z59/Csx7HSH/SIqFN2JWh0tAKAM6k/wRR0OZg== + "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18", "@types/express-serve-static-core@^4.17.21", "@types/express-serve-static-core@^4.17.5": version "4.17.21" resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz#a427278e106bca77b83ad85221eae709a3414d42" @@ -12604,6 +12609,13 @@ eventsource@^1.0.5, eventsource@^1.0.7: dependencies: original "^1.0.0" +eventsource@^1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/eventsource/-/eventsource-1.1.0.tgz#00e8ca7c92109e94b0ddf32dac677d841028cfaf" + integrity sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg== + dependencies: + original "^1.0.0" + evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"