Rewrite the /sync/:namespace/:kind/:name to return an event-stream

Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
2021-07-05 12:43:23 +02:00
parent c18e8eb910
commit f1200f44c8
15 changed files with 691 additions and 185 deletions
@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-techdocs': minor
+'@backstage/plugin-techdocs-backend': minor
+---
+
+Rewrite the `/sync/:namespace/:kind/:name` to return an event-stream.
+This change allows the sync process to take longer than a normal HTTP timeout.
+The stream also emits log events, so the caller can follow the build process in the frontend.
@@ -31,6 +31,7 @@
  },
  "dependencies": {
    "@backstage/backend-common": "^0.8.4",
+    "@backstage/catalog-client": "^0.3.15",
    "@backstage/catalog-model": "^0.8.4",
    "@backstage/config": "^0.1.5",
    "@backstage/errors": "^0.1.1",
@@ -47,6 +48,7 @@
  "devDependencies": {
    "@backstage/cli": "^0.7.3",
    "@types/dockerode": "^3.2.1",
+    "msw": "^0.29.0",
    "supertest": "^6.1.3"
  },
  "files": [
@@ -18,6 +18,7 @@ import {
  ENTITY_DEFAULT_NAMESPACE,
  stringifyEntityRef,
 } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
 import { NotModifiedError } from '@backstage/errors';
 import {
  GeneratorBase,
@@ -31,8 +32,8 @@ import {
 import fs from 'fs-extra';
 import os from 'os';
 import path from 'path';
+import { Writable } from 'stream';
 import { Logger } from 'winston';
-import { Config } from '@backstage/config';
 import { BuildMetadataStorage } from './BuildMetadataStorage';

 type DocsBuilderArguments = {
@@ -42,6 +43,7 @@ type DocsBuilderArguments = {
  entity: Entity;
  logger: Logger;
  config: Config;
+  logStream?: Writable;
 };

 export class DocsBuilder {
@@ -51,6 +53,7 @@ export class DocsBuilder {
  private entity: Entity;
  private logger: Logger;
  private config: Config;
+  private logStream: Writable | undefined;

  constructor({
    preparers,
@@ -59,6 +62,7 @@ export class DocsBuilder {
    entity,
    logger,
    config,
+    logStream,
  }: DocsBuilderArguments) {
    this.preparer = preparers.get(entity);
    this.generator = generators.get(entity);
@@ -66,6 +70,7 @@ export class DocsBuilder {
    this.entity = entity;
    this.logger = logger;
    this.config = config;
+    this.logStream = logStream;
  }

  /**
@@ -159,12 +164,14 @@ export class DocsBuilder {
    const outputDir = await fs.mkdtemp(
      path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
    );
+
    const parsedLocationAnnotation = getLocationForEntity(this.entity);
    await this.generator.run({
      inputDir: preparedDir,
      outputDir,
      parsedLocationAnnotation,
      etag: newEtag,
+      logStream: this.logStream,
    });

    // Remove Prepared directory since it is no longer needed.
@@ -0,0 +1,267 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  errorHandler,
+  getVoidLogger,
+  PluginEndpointDiscovery,
+} from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import {
+  GeneratorBuilder,
+  PreparerBuilder,
+  PublisherBase,
+} from '@backstage/techdocs-common';
+import express from 'express';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import request from 'supertest';
+import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
+import { createRouter } from './router';
+
+jest.mock('@backstage/config');
+jest.mock('../DocsBuilder');
+
+const MockedConfigReader = ConfigReader as jest.MockedClass<
+  typeof ConfigReader
+>;
+const MockedDocsBuilder = DocsBuilder as jest.MockedClass<typeof DocsBuilder>;
+
+const server = setupServer();
+
+describe('createRouter', () => {
+  // the calls from supertest should not be handled by msw so we only warn onUnhandledRequest
+  beforeAll(() => server.listen({ onUnhandledRequest: 'warn' }));
+  afterAll(() => server.close());
+  afterEach(() => server.resetHandlers());
+
+  const preparers: jest.Mocked<PreparerBuilder> = {
+    register: jest.fn(),
+    get: jest.fn(),
+  };
+  const generators: jest.Mocked<GeneratorBuilder> = {
+    register: jest.fn(),
+    get: jest.fn(),
+  };
+  const publisher: jest.Mocked<PublisherBase> = {
+    docsRouter: jest.fn(),
+    fetchTechDocsMetadata: jest.fn(),
+    getReadiness: jest.fn(),
+    hasDocsBeenGenerated: jest.fn(),
+    publish: jest.fn(),
+  };
+  const discovery: jest.Mocked<PluginEndpointDiscovery> = {
+    getBaseUrl: jest.fn(),
+    getExternalBaseUrl: jest.fn(),
+  };
+
+  let app: express.Express;
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  beforeEach(async () => {
+    publisher.docsRouter.mockReturnValue(() => {});
+    discovery.getBaseUrl.mockImplementation(async type => {
+      return `http://backstage.local/api/${type}`;
+    });
+
+    const router = await createRouter({
+      preparers,
+      generators,
+      publisher,
+      config: new ConfigReader({}),
+      logger: getVoidLogger(),
+      discovery,
+    });
+
+    router.use(errorHandler());
+    app = express();
+    app.use(router);
+  });
+
+  describe('GET /sync/:namespace/:kind/:name', () => {
+    it('should execute an update', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      MockedConfigReader.prototype.getString.mockReturnValue('local');
+
+      const entity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          uid: '0',
+          name: 'test',
+          namespace: 'default',
+          annotations: {
+            'sda.se/release-notes-location':
+              'github-releases:https://github.com/backstage/backstage',
+          },
+        },
+      };
+
+      server.use(
+        rest.get(
+          'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
+          (_req, res, ctx) => {
+            return res(ctx.json(entity));
+          },
+        ),
+      );
+
+      MockedDocsBuilder.prototype.build.mockImplementation(async () => {
+        // extract the logStream from the constructor call
+        const logStream = MockedDocsBuilder.mock.calls[0][0].logStream;
+
+        logStream?.write('Some log');
+        logStream?.write('Another log');
+
+        return true;
+      });
+
+      publisher.hasDocsBeenGenerated.mockResolvedValue(true);
+
+      const response = await request(app)
+        .get('/sync/default/Component/test')
+        .send();
+
+      expect(response.status).toBe(200);
+      expect(response.get('content-type')).toBe('text/event-stream');
+      expect(response.text).toEqual(
+        `event: log
+data: "Some log"
+
+event: log
+data: "Another log"
+
+event: finish
+data: {"updated":true}
+
+`,
+      );
+
+      expect(shouldCheckForUpdate).toBeCalledTimes(1);
+      expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
+      expect(DocsBuilder.prototype.build).toBeCalledTimes(1);
+    });
+
+    it('should not check for an update too often', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(false);
+
+      const entity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          uid: '0',
+          name: 'test',
+          namespace: 'default',
+          annotations: {
+            'sda.se/release-notes-location':
+              'github-releases:https://github.com/backstage/backstage',
+          },
+        },
+      };
+
+      server.use(
+        rest.get(
+          'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
+          (_req, res, ctx) => {
+            return res(ctx.json(entity));
+          },
+        ),
+      );
+
+      const response = await request(app)
+        .get('/sync/default/Component/test')
+        .send();
+
+      expect(response.status).toBe(200);
+      expect(response.get('content-type')).toBe('text/event-stream');
+      expect(response.text).toEqual(
+        `event: finish
+data: {"updated":false}
+
+`,
+      );
+
+      expect(shouldCheckForUpdate).toBeCalledTimes(1);
+      expect(MockedConfigReader.prototype.getString).toBeCalledTimes(0);
+      expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
+    });
+
+    it('should not check for an update without local builder', async () => {
+      (shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
+      MockedConfigReader.prototype.getString.mockReturnValue('external');
+
+      const entity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          uid: '0',
+          name: 'test',
+          namespace: 'default',
+          annotations: {
+            'sda.se/release-notes-location':
+              'github-releases:https://github.com/backstage/backstage',
+          },
+        },
+      };
+
+      server.use(
+        rest.get(
+          'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
+          (_req, res, ctx) => {
+            return res(ctx.json(entity));
+          },
+        ),
+      );
+
+      const response = await request(app)
+        .get('/sync/default/Component/test')
+        .send();
+
+      expect(response.status).toBe(200);
+      expect(response.get('content-type')).toBe('text/event-stream');
+      expect(response.text).toEqual(
+        `event: finish
+data: {"updated":false}
+
+`,
+      );
+
+      expect(shouldCheckForUpdate).toBeCalledTimes(1);
+      expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
+      expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
+    });
+
+    it('rejects when entity is not found', async () => {
+      server.use(
+        rest.get(
+          'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
+          (_req, res, ctx) => {
+            return res(ctx.status(404));
+          },
+        ),
+      );
+
+      const response = await request(app)
+        .get('/sync/default/Component/test')
+        .send();
+
+      expect(response.status).toBe(404);
+    });
+  });
+});
@@ -14,9 +14,10 @@
 * limitations under the License.
 */
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { CatalogClient } from '@backstage/catalog-client';
 import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import { NotFoundError } from '@backstage/errors';
 import {
  GeneratorBuilder,
  getLocationForEntity,
@@ -24,12 +25,12 @@ import {
  PublisherBase,
 } from '@backstage/techdocs-common';
 import fetch from 'cross-fetch';
-import express from 'express';
+import express, { Response } from 'express';
 import Router from 'express-promise-router';
 import { Knex } from 'knex';
+import { PassThrough } from 'stream';
 import { Logger } from 'winston';
-import { DocsBuilder } from '../DocsBuilder';
-import { shouldCheckForUpdate } from '../DocsBuilder/BuildMetadataStorage';
+import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';

 type RouterOptions = {
  preparers: PreparerBuilder;
@@ -50,6 +51,7 @@ export async function createRouter({
  discovery,
 }: RouterOptions): Promise<express.Router> {
  const router = Router();
+  const catalogClient = new CatalogClient({ discoveryApi: discovery });

  router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
    const { kind, namespace, name } = req.params;
@@ -108,56 +110,45 @@ export async function createRouter({
  });

  // Check if docs are the latest version and trigger rebuilds if not
-  // Responds with immediate success if rebuild not needed
+  // Responds with an event-stream that closes after the build finished
+  // Responds with an immediate success if rebuild not needed
  // If a build is required, responds with a success when finished
  router.get('/sync/:namespace/:kind/:name', async (req, res) => {
    const { kind, namespace, name } = req.params;
-    const catalogUrl = await discovery.getBaseUrl('catalog');
-    const triple = [kind, namespace, name].map(encodeURIComponent).join('/');
-
    const token = getBearerToken(req.headers.authorization);
-    const catalogRes = await fetch(`${catalogUrl}/entities/by-name/${triple}`, {
-      headers: token ? { Authorization: `Bearer ${token}` } : {},
-    });
-    if (!catalogRes.ok) {
-      const catalogResText = await catalogRes.text();
-      res.status(catalogRes.status);
-      res.send(catalogResText);
-      return;
-    }

-    const entity: Entity = await catalogRes.json();
+    const entity = await catalogClient.getEntityByName(
+      { kind, namespace, name },
+      { token },
+    );

-    if (!entity.metadata.uid) {
+    if (!entity?.metadata?.uid) {
      throw new NotFoundError('Entity metadata UID missing');
    }
+
+    // open the event-stream
+    const { log, error, finish } = createEventStream(res);
+
+    // create an in-memory stream to forward logs to the event-stream
+    const logStream = new PassThrough();
+    logStream.on('data', async data => {
+      log(data.toString().trim());
+    });
+
+    // check if the last update check was too recent
    if (!shouldCheckForUpdate(entity.metadata.uid)) {
-      res.status(200).json({
-        message: `Last check for documentation update is recent, did not retry.`,
-      });
+      finish({ updated: false });
      return;
    }

-    let publisherType = '';
-    try {
-      publisherType = config.getString('techdocs.publisher.type');
-    } catch (err) {
-      throw new Error(
-        'Unable to get techdocs.publisher.type in your app config. Set it to either ' +
-          "'local', 'googleGcs' or other support storage providers. Read more here " +
-          'https://backstage.io/docs/features/techdocs/architecture',
-      );
-    }
    // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local'
    // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
    // of the repository) is responsible for building and publishing documentation to the storage provider
    if (config.getString('techdocs.builder') !== 'local') {
-      res.status(200).json({
-        message:
-          '`techdocs.builder` app config is not set to `local`, so docs will not be generated locally and sync is not required.',
-      });
+      finish({ updated: false });
      return;
    }
+
    const docsBuilder = new DocsBuilder({
      preparers,
      generators,
@@ -165,52 +156,41 @@ export async function createRouter({
      logger,
      entity,
      config,
+      logStream,
    });
+
    let foundDocs = false;
-    switch (publisherType) {
-      case 'local':
-      case 'awsS3':
-      case 'azureBlobStorage':
-      case 'openStackSwift':
-      case 'googleGcs': {
-        // This block should be valid for all storage implementations. So no need to duplicate in future,
-        // add the publisher type in the list here.
-        const updated = await docsBuilder.build();

-        if (!updated) {
-          throw new NotModifiedError();
-        }
+    const updated = await docsBuilder.build();

-        // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched
-        // on the user's page. If not, respond with a message asking them to check back later.
-        // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second
-        for (let attempt = 0; attempt < 5; attempt++) {
-          if (await publisher.hasDocsBeenGenerated(entity)) {
-            foundDocs = true;
-            break;
-          }
-          await new Promise(r => setTimeout(r, 1000));
-        }
-        if (!foundDocs) {
-          logger.error(
-            'Published files are taking longer to show up in storage. Something went wrong.',
-          );
-          throw new NotFoundError(
-            'Sorry! It took too long for the generated docs to show up in storage. Check back later.',
-          );
-        }
+    if (!updated) {
+      finish({ updated: false });
+      return;
+    }

-        res
-          .status(201)
-          .json({ message: 'Docs updated or did not need updating' });
+    // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched
+    // on the user's page. If not, respond with a message asking them to check back later.
+    // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second
+    for (let attempt = 0; attempt < 5; attempt++) {
+      if (await publisher.hasDocsBeenGenerated(entity)) {
+        foundDocs = true;
        break;
      }
-
-      default:
-        throw new NotFoundError(
-          `Publisher type ${publisherType} is not supported by techdocs-backend docs builder.`,
-        );
+      await new Promise(r => setTimeout(r, 1000));
    }
+    if (!foundDocs) {
+      logger.error(
+        'Published files are taking longer to show up in storage. Something went wrong.',
+      );
+      error(
+        new NotFoundError(
+          'Sorry! It took too long for the generated docs to show up in storage. Check back later.',
+        ),
+      );
+      return;
+    }
+
+    finish({ updated: true });
  });

  // Route middleware which serves files from the storage set in the publisher.
@@ -222,3 +202,56 @@ export async function createRouter({
 function getBearerToken(header?: string): string | undefined {
  return header?.match(/(?:Bearer)\s+(\S+)/i)?.[1];
 }
+
+/**
+ * Create an event-stream response that emits the events 'log', 'error', and 'finish'.
+ *
+ * @param res the response to write the event-stream to
+ * @returns A tuple of <log, error, finish> callbacks to emit messages. A call to 'error' or 'finish'
+ *          will close the event-stream.
+ */
+function createEventStream(
+  res: Response<any, any>,
+): {
+  log: (message: string) => void;
+  error: (e: Error) => void;
+  finish: (result: { updated: boolean }) => void;
+} {
+  // Mandatory headers and http status to keep connection open
+  res.writeHead(200, {
+    Connection: 'keep-alive',
+    'Cache-Control': 'no-cache',
+    'Content-Type': 'text/event-stream',
+  });
+
+  // client closes connection
+  res.socket?.on('close', () => {
+    res.end();
+  });
+
+  // write the event to the stream
+  const send = (type: 'error' | 'finish' | 'log', data: any) => {
+    res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`);
+
+    // res.flush() is only available with the compression middleware
+    if (res.flush) {
+      res.flush();
+    }
+  };
+
+  return {
+    log: data => {
+      send('log', data);
+    },
+
+    error: e => {
+      send('error', e.message);
+      res.end();
+    },
+
+    finish: result => {
+      send('finish', result);
+      res.end();
+    },
+  };
+}
@@ -44,7 +44,7 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element;
 export const Router: () => JSX.Element;

 // @public (undocumented)
-export type SyncResult = 'cached' | 'updated' | 'timeout';
+export type SyncResult = 'cached' | 'updated';

 // @public (undocumented)
 export interface TechDocsApi {
@@ -112,7 +112,7 @@ export interface TechDocsStorageApi {
    // (undocumented)
    getStorageUrl(): Promise<string>;
    // (undocumented)
-    syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
+    syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise<SyncResult>;
 }

 // @public (undocumented)
@@ -140,7 +140,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
    getStorageUrl(): Promise<string>;
    // (undocumented)
    identityApi: IdentityApi;
-    syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
+    syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise<SyncResult>;
 }


@@ -17,6 +17,7 @@
 import { createDevApp } from '@backstage/dev-utils';
 import { NotFoundError } from '@backstage/errors';
 import React from 'react';
+import { EntityName } from '@backstage/catalog-model';
 import {
  Reader,
  SyncResult,
@@ -81,9 +82,18 @@ function createPage({
      });
    }

-    async syncEntityDocs() {
+    async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) {
      if (syncDocsDelay) {
-        await new Promise(resolve => setTimeout(resolve, syncDocsDelay));
+        for (let i = 0; i < 10; i++) {
+          setTimeout(
+            () => logHandler?.call(this, `Log line ${i}`),
+            ((i + 1) * syncDocsDelay) / 10,
+          );
+        }
+
+        await new Promise(resolve => {
+          setTimeout(resolve, syncDocsDelay);
+        });
      }

      return syncDocs();
@@ -195,13 +205,6 @@ createDevApp()
              syncDocsDelay: 2000,
            })}
          </TabbedLayout.Route>
-
-          <TabbedLayout.Route path="/timeout" title="Sync Timeout">
-            {createPage({
-              syncDocs: () => 'timeout',
-              syncDocsDelay: 2000,
-            })}
-          </TabbedLayout.Route>
        </TabbedLayout>
      </Page>
    ),
@@ -44,6 +44,7 @@
    "@material-ui/icons": "^4.9.1",
    "@material-ui/lab": "4.0.0-alpha.45",
    "@material-ui/styles": "^4.10.0",
+    "eventsource": "^1.1.0",
    "react": "^16.13.1",
    "react-dom": "^16.13.1",
    "react-router": "6.0.0-beta.0",
@@ -60,6 +61,7 @@
    "@testing-library/react": "^11.2.5",
    "@testing-library/react-hooks": "^3.4.2",
    "@testing-library/user-event": "^13.1.8",
+    "@types/eventsource": "^1.1.5",
    "@types/jest": "^26.0.7",
    "@types/node": "^14.14.32",
    "@types/react": "^16.9",
@@ -28,14 +28,17 @@ export const techdocsApiRef = createApiRef<TechDocsApi>({
  description: 'Used to make requests towards techdocs API',
 });

-export type SyncResult = 'cached' | 'updated' | 'timeout';
+export type SyncResult = 'cached' | 'updated';

 export interface TechDocsStorageApi {
  getApiOrigin(): Promise<string>;
  getStorageUrl(): Promise<string>;
  getBuilder(): Promise<string>;
  getEntityDocs(entityId: EntityName, path: string): Promise<string>;
-  syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
+  syncEntityDocs(
+    entityId: EntityName,
+    logHandler?: (line: string) => void,
+  ): Promise<SyncResult>;
  getBaseUrl(
    oldBaseUrl: string,
    entityId: EntityName,
@@ -13,9 +13,19 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
+
 import { Config } from '@backstage/config';
-import { TechDocsStorageClient } from './client';
 import { UrlPatternDiscovery } from '@backstage/core-app-api';
+import { IdentityApi } from '@backstage/core-plugin-api';
+import { NotFoundError } from '@backstage/errors';
+import EventSource from 'eventsource';
+import { TechDocsStorageClient } from './client';
+
+const MockedEventSource: jest.MockedClass<
+  typeof EventSource
+> = EventSource as any;
+
+jest.mock('eventsource');

 const mockEntity = {
  kind: 'Component',
@@ -29,6 +39,16 @@ describe('TechDocsStorageClient', () => {
    getOptionalString: () => 'http://backstage:9191/api/techdocs',
  } as Partial<Config>;
  const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
+  const identityApi: jest.Mocked<IdentityApi> = {
+    getIdToken: jest.fn(),
+    getProfile: jest.fn(),
+    getUserId: jest.fn(),
+    signOut: jest.fn(),
+  };
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });

  it('should return correct base url based on defined storage', async () => {
    // @ts-ignore Partial<Config> not assignable to Config.
@@ -51,4 +71,183 @@ describe('TechDocsStorageClient', () => {
      `${mockBaseUrl}/static/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`,
    );
  });
+
+  describe('syncEntityDocs', () => {
+    it('should create eventsource without headers', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      MockedEventSource.prototype.addEventListener.mockImplementation(
+        (type, fn) => {
+          if (type === 'finish') {
+            fn({ data: '{"updated": false}' } as any);
+          }
+        },
+      );
+
+      await storageApi.syncEntityDocs(mockEntity);
+
+      expect(
+        MockedEventSource,
+      ).toBeCalledWith(
+        'http://backstage:9191/api/techdocs/sync/default/Component/test-component',
+        { withCredentials: true, headers: {} },
+      );
+    });
+
+    it('should create eventsource with headers', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      MockedEventSource.prototype.addEventListener.mockImplementation(
+        (type, fn) => {
+          if (type === 'finish') {
+            fn({ data: '{"updated": false}' } as any);
+          }
+        },
+      );
+
+      identityApi.getIdToken.mockResolvedValue('token');
+
+      await storageApi.syncEntityDocs(mockEntity);
+
+      expect(
+        MockedEventSource,
+      ).toBeCalledWith(
+        'http://backstage:9191/api/techdocs/sync/default/Component/test-component',
+        { withCredentials: true, headers: { Authorization: 'Bearer token' } },
+      );
+    });
+
+    it('should resolve to cached', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      MockedEventSource.prototype.addEventListener.mockImplementation(
+        (type, fn) => {
+          if (type === 'finish') {
+            fn({ data: '{"updated": false}' } as any);
+          }
+        },
+      );
+
+      await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual(
+        'cached',
+      );
+    });
+
+    it('should resolve to updated', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      MockedEventSource.prototype.addEventListener.mockImplementation(
+        (type, fn) => {
+          if (type === 'finish') {
+            fn({ data: '{"updated": true}' } as any);
+          }
+        },
+      );
+
+      await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual(
+        'updated',
+      );
+    });
+
+    it('should log values', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      MockedEventSource.prototype.addEventListener.mockImplementation(
+        (type, fn) => {
+          if (type === 'log') {
+            fn({ data: '"A log message"' } as any);
+          }
+
+          if (type === 'finish') {
+            fn({ data: '{"updated": false}' } as any);
+          }
+        },
+      );
+
+      const logHandler = jest.fn();
+      await expect(
+        storageApi.syncEntityDocs(mockEntity, logHandler),
+      ).resolves.toEqual('cached');
+
+      expect(logHandler).toBeCalledTimes(1);
+      expect(logHandler).toBeCalledWith('A log message');
+    });
+
+    it('should throw NotFoundError', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      // we await later after we emitted the error
+      const promise = storageApi.syncEntityDocs(mockEntity).then();
+
+      // flush the event loop
+      await new Promise(setImmediate);
+
+      const instance = MockedEventSource.mock
+        .instances[0] as jest.Mocked<EventSource>;
+
+      instance.onerror({
+        status: 404,
+        message: 'Some not found warning',
+      } as any);
+
+      await expect(promise).rejects.toThrow(NotFoundError);
+      await expect(promise).rejects.toThrowError('Some not found warning');
+    });
+
+    it('should throw generic errors', async () => {
+      const storageApi = new TechDocsStorageClient({
+        // @ts-ignore Partial<Config> not assignable to Config.
+        configApi,
+        discoveryApi,
+        identityApi,
+      });
+
+      // we await later after we emitted the error
+      const promise = storageApi.syncEntityDocs(mockEntity).then();
+
+      // flush the event loop
+      await new Promise(setImmediate);
+
+      const instance = MockedEventSource.mock
+        .instances[0] as jest.Mocked<EventSource>;
+
+      instance.onerror({
+        status: 500,
+        message: 'Some other error',
+      } as any);
+
+      await expect(promise).rejects.toThrow(Error);
+      await expect(promise).rejects.toThrowError('Some other error');
+    });
+  });
 });
@@ -16,10 +16,11 @@

 import { EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
+import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api';
 import { NotFoundError } from '@backstage/errors';
+import EventSource from 'eventsource';
 import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api';
 import { TechDocsEntityMetadata, TechDocsMetadata } from './types';
-import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api';

 /**
 * API to talk to techdocs-backend.
@@ -192,44 +193,59 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
   * Check if docs are on the latest version and trigger rebuild if not
   *
   * @param {EntityName} entityId Object containing entity data like name, namespace, etc.
+   * @param {Function} logHandler Callback to receive log messages from the build process
   * @returns {SyncResult} Whether documents are currently synchronized to newest version
   * @throws {Error} Throws error on error from sync endpoint in Techdocs Backend
   */
-  async syncEntityDocs(entityId: EntityName): Promise<SyncResult> {
+  async syncEntityDocs(
+    entityId: EntityName,
+    logHandler: (line: string) => void = () => {},
+  ): Promise<SyncResult> {
    const { kind, namespace, name } = entityId;

    const apiOrigin = await this.getApiOrigin();
    const url = `${apiOrigin}/sync/${namespace}/${kind}/${name}`;
    const token = await this.identityApi.getIdToken();
-    let request;
-    let attempts: number = 0;
-    // retry if request times out, up to 5 times
-    // can happen due to docs taking too long to generate
-    while (!request || (request.status === 408 && attempts < 5)) {
-      attempts++;
-      request = await fetch(url, {
+
+    return new Promise((resolve, reject) => {
+      const source = new EventSource(url, {
+        withCredentials: true,
        headers: token ? { Authorization: `Bearer ${token}` } : {},
      });
-    }

-    switch (request.status) {
-      case 404:
-        throw new NotFoundError((await request.json()).error);
+      source.addEventListener('log', (e: any) => {
+        if (e.data) {
+          logHandler(JSON.parse(e.data));
+        }
+      });

-      case 200:
-      case 304:
-        return 'cached';
+      source.addEventListener('finish', (e: any) => {
+        let updated: boolean = false;

-      case 201:
-        return 'updated';
+        if (e.data) {
+          ({ updated } = JSON.parse(e.data));
+        }

-      // for timeout and misc errors, handle without error to allow viewing older docs
-      // if older docs not available,
-      // Reader will show 404 error coming from getEntityDocs
-      case 408:
-      default:
-        return 'timeout';
-    }
+        resolve(updated ? 'updated' : 'cached');
+      });
+
+      source.onerror = (e: any) => {
+        source.close();
+
+        switch (e.status) {
+          // the endpoint returned a 404 status
+          case 404:
+            reject(new NotFoundError(e.message));
+            return;
+
+          // also handles the event-stream close. the reject is ignored if the Promise was already
+          // resolved by a finish event.
+          default:
+            reject(new Error(e.message));
+            return;
+        }
+      };
+    });
  }

  async getBaseUrl(
@@ -15,6 +15,7 @@
 */

 import { EntityName } from '@backstage/catalog-model';
+import { useApi } from '@backstage/core-plugin-api';
 import { scmIntegrationsApiRef } from '@backstage/integration-react';
 import { BackstageTheme } from '@backstage/theme';
 import { useTheme } from '@material-ui/core';
@@ -37,7 +38,6 @@ import {
 import { TechDocsNotFound } from './TechDocsNotFound';
 import TechDocsProgressBar from './TechDocsProgressBar';
 import { useReaderState } from './useReaderState';
-import { useApi } from '@backstage/core-plugin-api';

 type Props = {
  entityId: EntityName;
@@ -328,12 +328,6 @@ export const Reader = ({ entityId, onReady }: Props) => {
          to view.
        </Alert>
      )}
-      {state === 'CONTENT_STALE_TIMEOUT' && (
-        <Alert variant="outlined" severity="warning">
-          Building a newer version of this documentation took longer than
-          expected. Please refresh to try again.
-        </Alert>
-      )}
      {state === 'CONTENT_STALE_ERROR' && (
        <Alert variant="outlined" severity="error">
          Building a newer version of this documentation failed. {errorMessage}
@@ -14,6 +14,7 @@
 * limitations under the License.
 */

+import { ApiProvider, ApiRegistry } from '@backstage/core-app-api';
 import { NotFoundError } from '@backstage/errors';
 import { act, renderHook } from '@testing-library/react-hooks';
 import React from 'react';
@@ -23,7 +24,6 @@ import {
  reducer,
  useReaderState,
 } from './useReaderState';
-import { ApiProvider, ApiRegistry } from '@backstage/core-app-api';

 describe('useReaderState', () => {
  let Wrapper: React.ComponentType;
@@ -55,14 +55,12 @@ describe('useReaderState', () => {
      ${false}       | ${undefined} | ${'BUILDING'}           | ${'INITIAL_BUILD'}
      ${false}       | ${undefined} | ${'BUILD_READY'}        | ${'CONTENT_NOT_FOUND'}
      ${false}       | ${undefined} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'}
-      ${false}       | ${undefined} | ${'BUILD_TIMED_OUT'}    | ${'CONTENT_NOT_FOUND'}
      ${false}       | ${undefined} | ${'UP_TO_DATE'}         | ${'CONTENT_NOT_FOUND'}
      ${false}       | ${undefined} | ${'ERROR'}              | ${'CONTENT_NOT_FOUND'}
      ${false}       | ${'asdf'}    | ${'CHECKING'}           | ${'CONTENT_FRESH'}
      ${false}       | ${'asdf'}    | ${'BUILDING'}           | ${'CONTENT_STALE_REFRESHING'}
      ${false}       | ${'asdf'}    | ${'BUILD_READY'}        | ${'CONTENT_STALE_READY'}
      ${false}       | ${'asdf'}    | ${'BUILD_READY_RELOAD'} | ${'CHECKING'}
-      ${false}       | ${'asdf'}    | ${'BUILD_TIMED_OUT'}    | ${'CONTENT_STALE_TIMEOUT'}
      ${false}       | ${'asdf'}    | ${'UP_TO_DATE'}         | ${'CONTENT_FRESH'}
      ${false}       | ${'asdf'}    | ${'ERROR'}              | ${'CONTENT_STALE_ERROR'}
    `(
@@ -369,42 +367,6 @@ describe('useReaderState', () => {
      });
    });

-    it('should handle timed-out refresh', async () => {
-      techdocsStorageApi.getEntityDocs.mockResolvedValue('my content');
-      techdocsStorageApi.syncEntityDocs.mockResolvedValue('timeout');
-
-      await act(async () => {
-        const { result, waitForValueToChange } = await renderHook(
-          () => useReaderState('Component', 'default', 'backstage', '/example'),
-          { wrapper: Wrapper },
-        );
-
-        expect(result.current).toEqual({
-          state: 'CHECKING',
-          content: undefined,
-          errorMessage: '',
-        });
-
-        // the content is returned but the sync is in progress
-        await waitForValueToChange(() => result.current.state);
-        expect(result.current).toEqual({
-          state: 'CONTENT_STALE_TIMEOUT',
-          content: 'my content',
-          errorMessage: '',
-        });
-
-        expect(techdocsStorageApi.getEntityDocs).toBeCalledWith(
-          { kind: 'Component', namespace: 'default', name: 'backstage' },
-          '/example',
-        );
-        expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({
-          kind: 'Component',
-          namespace: 'default',
-          name: 'backstage',
-        });
-      });
-    });
-
    it('should handle content error', async () => {
      techdocsStorageApi.getEntityDocs.mockRejectedValue(
        new NotFoundError('Some error description'),
@@ -14,10 +14,10 @@
 * limitations under the License.
 */

+import { useApi } from '@backstage/core-plugin-api';
 import { useEffect, useMemo, useReducer, useRef } from 'react';
 import { useAsync, useAsyncRetry } from 'react-use';
 import { techdocsStorageApiRef } from '../../api';
-import { useApi } from '@backstage/core-plugin-api';

 /**
 * A state representation that is used to configure the UI of <Reader />
@@ -35,9 +35,6 @@ type ContentStateTypes =
  /** There is content, but after a reload, the content will be different */
  | 'CONTENT_STALE_READY'

-  /** There is content, the backend tried to update it, but it took too long */
-  | 'CONTENT_STALE_TIMEOUT'
-
  /** There is content, the backend tried to update it, but failed */
  | 'CONTENT_STALE_ERROR'

@@ -93,11 +90,6 @@ export function calculateDisplayState({
    return 'CONTENT_STALE_READY';
  }

-  // the build timed out, but the content is still stale
-  if (activeSyncState === 'BUILD_TIMED_OUT') {
-    return 'CONTENT_STALE_TIMEOUT';
-  }
-
  // the build failed, but the content is still stale
  if (activeSyncState === 'ERROR') {
    return 'CONTENT_STALE_ERROR';
@@ -127,9 +119,6 @@ type SyncStates =
   */
  | 'BUILD_READY_RELOAD'

-  /** Building the documentation timed out */
-  | 'BUILD_TIMED_OUT'
-
  /** No need for a sync. The content was already up-to-date. */
  | 'UP_TO_DATE'

@@ -274,18 +263,27 @@ export function useReaderState(
        name,
      });

-      if (result === 'updated') {
-        // if there was no content prior to building, retry the loading
-        if (!contentRef.current.content) {
-          contentRef.current.reload();
-          dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' });
-        } else {
-          dispatch({ type: 'sync', state: 'BUILD_READY' });
-        }
-      } else if (result === 'cached') {
-        dispatch({ type: 'sync', state: 'UP_TO_DATE' });
-      } else {
-        dispatch({ type: 'sync', state: 'BUILD_TIMED_OUT' });
+      switch (result) {
+        case 'updated':
+          // if there was no content prior to building, retry the loading
+          if (!contentRef.current.content) {
+            contentRef.current.reload();
+            dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' });
+          } else {
+            dispatch({ type: 'sync', state: 'BUILD_READY' });
+          }
+          break;
+        case 'cached':
+          dispatch({ type: 'sync', state: 'UP_TO_DATE' });
+          break;
+
+        default:
+          dispatch({
+            type: 'sync',
+            state: 'ERROR',
+            syncError: new Error('Unexpected return state'),
+          });
+          break;
      }
    } catch (e) {
      dispatch({ type: 'sync', state: 'ERROR', syncError: e });
@@ -5745,6 +5745,11 @@
  resolved "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
  integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==

+"@types/eventsource@^1.1.5":
+  version "1.1.5"
+  resolved "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.5.tgz#408e9b45efb176c8bea672ab58c81e7ab00d24bc"
+  integrity sha512-BA9q9uC2PAMkUS7DunHTxWZZaVpeNzDG8lkBxcKwzKJClfDQ4Z59/Csx7HSH/SIqFN2JWh0tAKAM6k/wRR0OZg==
+
 "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18", "@types/express-serve-static-core@^4.17.21", "@types/express-serve-static-core@^4.17.5":
  version "4.17.21"
  resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz#a427278e106bca77b83ad85221eae709a3414d42"
@@ -12604,6 +12609,13 @@ eventsource@^1.0.5, eventsource@^1.0.7:
  dependencies:
    original "^1.0.0"

+eventsource@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.npmjs.org/eventsource/-/eventsource-1.1.0.tgz#00e8ca7c92109e94b0ddf32dac677d841028cfaf"
+  integrity sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg==
+  dependencies:
+    original "^1.0.0"
+
 evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3:
  version "1.0.3"
  resolved "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"