Merge branch 'backstage:master' into feat/retry-in-useentity

plugins/kubernetes-backend/api-report.md

@@ -10,6 +10,14 @@ import { KubernetesFetchError } from '@backstage/plugin-kubernetes-common';
 import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common';
 import { Logger as Logger_2 } from 'winston';
 
+// Warning: (ae-missing-release-tag) "AWSClusterDetails" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface AWSClusterDetails extends ClusterDetails {
+  // (undocumented)
+  assumeRole?: string;
+}
+
 // Warning: (ae-missing-release-tag) "ClusterDetails" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -53,6 +61,11 @@ export interface FetchResponseWrapper {
   responses: FetchResponse[];
 }
 
+// Warning: (ae-missing-release-tag) "GKEClusterDetails" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface GKEClusterDetails extends ClusterDetails {}
+
 // Warning: (ae-missing-release-tag) "KubernetesClustersSupplier" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -107,7 +120,11 @@ export const makeRouter: (
 // @public (undocumented)
 export interface ObjectFetchParams {
   // (undocumented)
-  clusterDetails: ClusterDetails;
+  clusterDetails:
+    | AWSClusterDetails
+    | GKEClusterDetails
+    | ServiceAccountClusterDetails
+    | ClusterDetails;
   // (undocumented)
   customResources: CustomResource[];
   // (undocumented)
@@ -130,6 +147,11 @@ export interface RouterOptions {
   logger: Logger_2;
 }
 
+// Warning: (ae-missing-release-tag) "ServiceAccountClusterDetails" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface ServiceAccountClusterDetails extends ClusterDetails {}
+
 // Warning: (ae-missing-release-tag) "ServiceLocatorMethod" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/kubernetes-backend/package.json

@@ -55,7 +55,9 @@
   "devDependencies": {
     "@backstage/cli": "^0.7.4",
     "@types/aws4": "^1.5.1",
-    "supertest": "^6.1.3"
+    "supertest": "^6.1.3",
+    "aws-sdk-mock": "^5.2.1",
+    "bdd-lazy-var": "^2.6.0"
   },
   "files": [
     "dist",

plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts

@@ -97,4 +97,48 @@ describe('ConfigClusterLocator', () => {
       },
     ]);
   });
+
+  it('one aws cluster with assumeRole and one without', async () => {
+    const config: Config = new ConfigReader({
+      clusters: [
+        {
+          name: 'cluster1',
+          serviceAccountToken: 'token',
+          url: 'http://localhost:8080',
+          authProvider: 'aws',
+          skipTLSVerify: false,
+        },
+        {
+          assumeRole: 'SomeRole',
+          name: 'cluster2',
+          url: 'http://localhost:8081',
+          authProvider: 'aws',
+          skipTLSVerify: true,
+        },
+      ],
+    });
+
+    const sut = ConfigClusterLocator.fromConfig(config);
+
+    const result = await sut.getClusters();
+
+    expect(result).toStrictEqual([
+      {
+        assumeRole: undefined,
+        name: 'cluster1',
+        serviceAccountToken: 'token',
+        url: 'http://localhost:8080',
+        authProvider: 'aws',
+        skipTLSVerify: false,
+      },
+      {
+        assumeRole: 'SomeRole',
+        name: 'cluster2',
+        serviceAccountToken: undefined,
+        url: 'http://localhost:8081',
+        authProvider: 'aws',
+        skipTLSVerify: true,
+      },
+    ]);
+  });
 });

plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts

@@ -29,13 +29,32 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier {
     // is required if authProvider is serviceAccount
     return new ConfigClusterLocator(
       config.getConfigArray('clusters').map(c => {
-        return {
+        const authProvider = c.getString('authProvider');
+        const clusterDetails = {
           name: c.getString('name'),
           url: c.getString('url'),
           serviceAccountToken: c.getOptionalString('serviceAccountToken'),
           skipTLSVerify: c.getOptionalBoolean('skipTLSVerify') ?? false,
-          authProvider: c.getString('authProvider'),
+          authProvider: authProvider,
         };
+
+        switch (authProvider) {
+          case 'google': {
+            return clusterDetails;
+          }
+          case 'aws': {
+            const assumeRole = c.getOptionalString('assumeRole');
+            return { assumeRole, ...clusterDetails };
+          }
+          case 'serviceAccount': {
+            return clusterDetails;
+          }
+          default: {
+            throw new Error(
+              `authProvider "${authProvider}" has no config associated with it`,
+            );
+          }
+        }
       }),
     );
   }

plugins/kubernetes-backend/src/cluster-locator/GkeClusterLocator.ts

@@ -16,7 +16,7 @@
 
 import { Config } from '@backstage/config';
 import * as container from '@google-cloud/container';
-import { ClusterDetails, KubernetesClustersSupplier } from '../types/types';
+import { GKEClusterDetails, KubernetesClustersSupplier } from '../types/types';
 
 type GkeClusterLocatorOptions = {
   projectId: string;
@@ -49,7 +49,7 @@ export class GkeClusterLocator implements KubernetesClustersSupplier {
     );
   }
 
-  async getClusters(): Promise<ClusterDetails[]> {
+  async getClusters(): Promise<GKEClusterDetails[]> {
     const { projectId, region, skipTLSVerify } = this.options;
     const request = {
       parent: `projects/${projectId}/locations/${region}`,

plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.test.ts

@@ -14,15 +14,58 @@
  * limitations under the License.
  */
 import AWS from 'aws-sdk';
+import AWSMock from 'aws-sdk-mock';
 import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator';
+import { get, def } from 'bdd-lazy-var';
 
 describe('AwsIamKubernetesAuthTranslator tests', () => {
+  let role: any = undefined;
+  const credentials: any = {
+    accessKeyId: 'bloop',
+    secretAccessKey: 'omg-so-secret',
+    sessionToken: 'token',
+  };
+
+  let assumeResponse: any = {
+    Credentials: {
+      AccessKeyId: credentials.accessKeyId,
+      SecretAccessKey: credentials.secretAccessKey,
+      SessionToken: credentials.sessionToken,
+    },
+  };
+
+  let credentialsResponse: any = new AWS.Credentials(credentials);
+
+  AWSMock.setSDKInstance(AWS);
+
   beforeEach(() => {
     jest.resetAllMocks();
   });
-  it('returns a signed url for aws credentials', async () => {
+
+  afterAll(() => {
+    jest.resetAllMocks();
+  });
+
+  def('subject', () => {
+    AWSMock.mock('STS', 'assumeRole', (_params: any, callback: Function) => {
+      callback(null, assumeResponse);
+    });
+
     const authTranslator = new AwsIamKubernetesAuthTranslator();
 
+    jest
+      .spyOn(authTranslator, 'awsGetCredentials')
+      .mockImplementation(async () => credentialsResponse);
+
+    return authTranslator.decorateClusterDetailsWithAuth({
+      assumeRole: role,
+      name: 'test-cluster',
+      url: '',
+      authProvider: 'aws',
+    });
+  });
+
+  it('returns a signed url for aws credentials', async () => {
     // These credentials are not real.
     // Pulled from example in docs: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html
     AWS.config.credentials = new AWS.Credentials(
@@ -30,24 +73,47 @@ describe('AwsIamKubernetesAuthTranslator tests', () => {
       'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
     );
 
-    const clusterDetails = await authTranslator.decorateClusterDetailsWithAuth({
-      name: 'test-cluster',
-      url: '',
-      authProvider: 'aws',
-    });
-    expect(clusterDetails.serviceAccountToken).toBeDefined();
+    const subject = await get('subject');
+    expect(subject.serviceAccountToken).toBeDefined();
   });
 
-  it('throws when unable to get aws credentials', async () => {
-    AWS.config.credentials = undefined;
-    const authTranslator = new AwsIamKubernetesAuthTranslator();
-    const promise = authTranslator.decorateClusterDetailsWithAuth({
-      name: 'test-cluster',
-      url: '',
-      authProvider: 'aws',
-    });
-    await expect(promise).rejects.toThrow(
-      'Could not load credentials from any providers',
+  describe('When the role is assumed', () => {
+    // These credentials are not real.
+    // Pulled from example in docs: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html
+    AWS.config.credentials = new AWS.Credentials(
+      'AKIAIOSFODNN7EXAMPLE',
+      'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
     );
+    role = 'SomeRole';
+
+    describe('When the role is valid', () => {
+      it('returns a signed url for aws credentials', async () => {
+        const subject = await get('subject');
+        expect(subject.serviceAccountToken).toBeDefined();
+      });
+    });
+
+    describe('When the role is invalid', () => {
+      it('returns the original AWS credentials', async () => {
+        assumeResponse = undefined;
+        await expect(get('subject')).rejects.toThrow(/Unable to assume role:/);
+      });
+    });
+  });
+
+  describe('When no creds are returned from AWS', () => {
+    it('throws unable to get aws credentials', async () => {
+      credentialsResponse = new Error();
+      await expect(get('subject')).rejects.toThrow('No AWS credentials found.');
+    });
+  });
+
+  describe('When invalid creds are returned from AWS', () => {
+    it('throws credentials are invalid to get aws credentials', async () => {
+      credentialsResponse = new AWS.Credentials(credentialsResponse);
+      await expect(get('subject')).rejects.toThrow(
+        'Invalid AWS credentials found.',
+      );
+    });
   });
 });

plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.ts

@@ -15,7 +15,7 @@
  */
 import AWS, { Credentials } from 'aws-sdk';
 import { sign } from 'aws4';
-import { ClusterDetails } from '../types/types';
+import { AWSClusterDetails } from '../types/types';
 import { KubernetesAuthTranslator } from './types';
 
 const base64 = (str: string) =>
@@ -29,23 +29,78 @@ const pipe = (fns: ReadonlyArray<any>) => (thing: string): string =>
 const removePadding = replace(/=+$/, '');
 const makeUrlSafe = pipe([replace('+', '-'), replace('/', '_')]);
 
+type SigningCreds = {
+  accessKeyId: string | undefined;
+  secretAccessKey: string | undefined;
+  sessionToken: string | undefined;
+};
+
 export class AwsIamKubernetesAuthTranslator
   implements KubernetesAuthTranslator {
-  async getBearerToken(clusterName: string): Promise<string> {
-    const credentials = await new Promise((resolve, reject) => {
+  validCredentials(creds: SigningCreds): boolean {
+    return ((creds?.accessKeyId &&
+      creds?.secretAccessKey &&
+      creds?.sessionToken) as unknown) as boolean;
+  }
+
+  awsGetCredentials = async (): Promise<Credentials> => {
+    return new Promise((resolve, reject) => {
       AWS.config.getCredentials(err => {
         if (err) {
-          reject(err);
-        } else {
-          resolve(AWS.config.credentials);
+          return reject(err);
         }
+
+        return resolve(AWS.config.credentials as Credentials);
       });
     });
+  };
+
+  async getCredentials(assumeRole: string | undefined): Promise<SigningCreds> {
+    return new Promise<SigningCreds>(async (resolve, reject) => {
+      const awsCreds = await this.awsGetCredentials();
+
+      if (!(awsCreds instanceof Credentials))
+        return reject(Error('No AWS credentials found.'));
+
+      let creds: SigningCreds = {
+        accessKeyId: awsCreds.accessKeyId,
+        secretAccessKey: awsCreds.secretAccessKey,
+        sessionToken: awsCreds.sessionToken,
+      };
+
+      if (!this.validCredentials(creds))
+        return reject(Error('Invalid AWS credentials found.'));
+      if (!assumeRole) return resolve(creds);
+
+      try {
+        const params = {
+          RoleArn: assumeRole,
+          RoleSessionName: 'backstage-login',
+        };
+        const assumedRole = await new AWS.STS().assumeRole(params).promise();
+
+        if (!assumedRole.Credentials) {
+          throw new Error(`No credentials returned for role ${assumeRole}`);
+        }
+
+        creds = {
+          accessKeyId: assumedRole.Credentials.AccessKeyId,
+          secretAccessKey: assumedRole.Credentials.SecretAccessKey,
+          sessionToken: assumedRole.Credentials.SessionToken,
+        };
+      } catch (e) {
+        console.warn(`There was an error assuming the role: ${e}`);
+        return reject(Error(`Unable to assume role: ${e}`));
+      }
+      return resolve(creds);
+    });
+  }
+  async getBearerToken(
+    clusterName: string,
+    assumeRole: string | undefined,
+  ): Promise<string> {
+    const credentials = await this.getCredentials(assumeRole);
 
-    if (!(credentials instanceof Credentials)) {
-      throw new Error('no AWS credentials found.');
-    }
-    await credentials.getPromise();
     const request = {
       host: `sts.amazonaws.com`,
       path: `/?Action=GetCallerIdentity&Version=2011-06-15&X-Amz-Expires=60`,
@@ -54,11 +109,8 @@ export class AwsIamKubernetesAuthTranslator
       },
       signQuery: true,
     };
-    const signedRequest = sign(request, {
-      accessKeyId: credentials.accessKeyId,
-      secretAccessKey: credentials.secretAccessKey,
-      sessionToken: credentials.sessionToken,
-    });
+
+    const signedRequest = sign(request, credentials);
 
     return pipe([
       (signed: any) => `https://${signed.host}${signed.path}`,
@@ -70,15 +122,16 @@ export class AwsIamKubernetesAuthTranslator
   }
 
   async decorateClusterDetailsWithAuth(
-    clusterDetails: ClusterDetails,
-  ): Promise<ClusterDetails> {
-    const clusterDetailsWithAuthToken: ClusterDetails = Object.assign(
+    clusterDetails: AWSClusterDetails,
+  ): Promise<AWSClusterDetails> {
+    const clusterDetailsWithAuthToken: AWSClusterDetails = Object.assign(
       {},
       clusterDetails,
     );
 
     clusterDetailsWithAuthToken.serviceAccountToken = await this.getBearerToken(
       clusterDetails.name,
+      clusterDetails.assumeRole,
     );
     return clusterDetailsWithAuthToken;
   }

plugins/kubernetes-backend/src/kubernetes-auth-translator/GoogleKubernetesAuthTranslator.ts

@@ -15,16 +15,16 @@
  */
 
 import { KubernetesAuthTranslator } from './types';
-import { ClusterDetails } from '../types/types';
+import { GKEClusterDetails } from '../types/types';
 import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common';
 
 export class GoogleKubernetesAuthTranslator
   implements KubernetesAuthTranslator {
   async decorateClusterDetailsWithAuth(
-    clusterDetails: ClusterDetails,
+    clusterDetails: GKEClusterDetails,
     requestBody: KubernetesRequestBody,
-  ): Promise<ClusterDetails> {
-    const clusterDetailsWithAuthToken: ClusterDetails = Object.assign(
+  ): Promise<GKEClusterDetails> {
+    const clusterDetailsWithAuthToken: GKEClusterDetails = Object.assign(
       {},
       clusterDetails,
     );

plugins/kubernetes-backend/src/kubernetes-auth-translator/ServiceAccountKubernetesAuthTranslator.ts

@@ -15,18 +15,18 @@
  */
 
 import { KubernetesAuthTranslator } from './types';
-import { ClusterDetails } from '../types/types';
+import { ServiceAccountClusterDetails } from '../types/types';
 import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common';
 
 export class ServiceAccountKubernetesAuthTranslator
   implements KubernetesAuthTranslator {
   async decorateClusterDetailsWithAuth(
-    clusterDetails: ClusterDetails,
+    clusterDetails: ServiceAccountClusterDetails,
     // To ignore TS6133 linting error where it detects 'requestBody' is declared but its value is never read.
     // @ts-ignore-start
     requestBody: KubernetesRequestBody, // eslint-disable-line @typescript-eslint/no-unused-vars
     // @ts-ignore-end
-  ): Promise<ClusterDetails> {
+  ): Promise<ServiceAccountClusterDetails> {
     return clusterDetails;
   }
 }

plugins/kubernetes-backend/src/types/types.ts

@@ -27,7 +27,11 @@ export interface CustomResource {
 
 export interface ObjectFetchParams {
   serviceId: string;
-  clusterDetails: ClusterDetails;
+  clusterDetails:
+    | AWSClusterDetails
+    | GKEClusterDetails
+    | ServiceAccountClusterDetails
+    | ClusterDetails;
   objectTypesToFetch: Set<KubernetesObjectTypes>;
   labelSelector: string;
   customResources: CustomResource[];
@@ -77,3 +81,9 @@ export interface ClusterDetails {
   serviceAccountToken?: string | undefined;
   skipTLSVerify?: boolean;
 }
+
+export interface GKEClusterDetails extends ClusterDetails {}
+export interface ServiceAccountClusterDetails extends ClusterDetails {}
+export interface AWSClusterDetails extends ClusterDetails {
+  assumeRole?: string;
+}

plugins/scaffolder-backend/api-report.md

@@ -107,6 +107,13 @@ export const createFilesystemDeleteAction: () => TemplateAction<any>;
 // @public (undocumented)
 export const createFilesystemRenameAction: () => TemplateAction<any>;
 
+// Warning: (ae-missing-release-tag) "createGithubActionsDispatchAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export function createGithubActionsDispatchAction(options: {
+  integrations: ScmIntegrationRegistry;
+}): TemplateAction<any>;
+
 // Warning: (ae-missing-release-tag) "createPublishAzureAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/scaffolder-backend/src/scaffolder/__mocks__/@octokit/rest/index.ts

@@ -15,6 +15,11 @@
  */
 
 export const mockGithubClient = {
+  rest: {
+    actions: {
+      createWorkflowDispatch: jest.fn(),
+    },
+  },
   repos: {
     createInOrg: jest.fn(),
     createForAuthenticatedUser: jest.fn(),

plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts

@@ -37,6 +37,7 @@ import {
   createPublishGithubPullRequestAction,
   createPublishGitlabAction,
 } from './publish';
+import { createGithubActionsDispatchAction } from './github';
 
 export const createBuiltinActions = (options: {
   reader: UrlReader;
@@ -91,5 +92,8 @@ export const createBuiltinActions = (options: {
     createCatalogWriteAction(),
     createFilesystemDeleteAction(),
     createFilesystemRenameAction(),
+    createGithubActionsDispatchAction({
+      integrations,
+    }),
   ];
 };

plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('@octokit/rest');
+
+import { createGithubActionsDispatchAction } from './githubActionsDispatch';
+import { ScmIntegrations } from '@backstage/integration';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '@backstage/backend-common';
+import { PassThrough } from 'stream';
+
+describe('github:actions:dispatch', () => {
+  const config = new ConfigReader({
+    integrations: {
+      github: [
+        { host: 'github.com', token: 'tokenlols' },
+        { host: 'ghe.github.com' },
+      ],
+    },
+  });
+
+  const integrations = ScmIntegrations.fromConfig(config);
+  const action = createGithubActionsDispatchAction({ integrations });
+
+  const mockContext = {
+    input: {
+      repoUrl: 'github.com?repo=repo&owner=owner',
+      workflowId: 'a-workflow-id',
+      branchOrTagName: 'main',
+    },
+    workspacePath: 'lol',
+    logger: getVoidLogger(),
+    logStream: new PassThrough(),
+    output: jest.fn(),
+    createTemporaryDirectory: jest.fn(),
+  };
+
+  const { mockGithubClient } = require('@octokit/rest');
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  it('should throw an error when the repoUrl is not well formed', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: { repoUrl: 'github.com?repo=bob' },
+      }),
+    ).rejects.toThrow(/missing owner/);
+
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: { repoUrl: 'github.com?owner=owner' },
+      }),
+    ).rejects.toThrow(/missing repo/);
+  });
+
+  it('should throw if there is no integration config provided', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: { repoUrl: 'missing.com?repo=bob&owner=owner' },
+      }),
+    ).rejects.toThrow(/No matching integration configuration/);
+  });
+
+  it('should throw if there is no token in the integration config that is returned', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: {
+          repoUrl: 'ghe.github.com?repo=bob&owner=owner',
+        },
+      }),
+    ).rejects.toThrow(/No token available for host/);
+  });
+
+  it('should call the githubApis for creating WorkflowDispatch', async () => {
+    mockGithubClient.rest.actions.createWorkflowDispatch.mockResolvedValue({
+      data: {
+        foo: 'bar',
+      },
+    });
+
+    const repoUrl = 'github.com?repo=repo&owner=owner';
+    const workflowId = 'dispatch_workflow';
+    const branchOrTagName = 'main';
+    const ctx = Object.assign({}, mockContext, {
+      input: { repoUrl, workflowId, branchOrTagName },
+    });
+    await action.handler(ctx);
+
+    expect(
+      mockGithubClient.rest.actions.createWorkflowDispatch,
+    ).toHaveBeenCalledWith({
+      owner: 'owner',
+      repo: 'repo',
+      workflow_id: workflowId,
+      ref: branchOrTagName,
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { InputError } from '@backstage/errors';
+import {
+  GithubCredentialsProvider,
+  ScmIntegrationRegistry,
+} from '@backstage/integration';
+import { Octokit } from '@octokit/rest';
+import { parseRepoUrl } from '../publish/util';
+import { createTemplateAction } from '../../createTemplateAction';
+
+export function createGithubActionsDispatchAction(options: {
+  integrations: ScmIntegrationRegistry;
+}) {
+  const { integrations } = options;
+
+  const credentialsProviders = new Map(
+    integrations.github.list().map(integration => {
+      const provider = GithubCredentialsProvider.create(integration.config);
+      return [integration.config.host, provider];
+    }),
+  );
+
+  return createTemplateAction<{
+    repoUrl: string;
+    workflowId: string;
+    branchOrTagName: string;
+  }>({
+    id: 'github:actions:dispatch',
+    description:
+      'Dispatches a GitHub Action workflow for a given branch or tag',
+    schema: {
+      input: {
+        type: 'object',
+        required: ['repoUrl', 'workflowId', 'branchOrTagName'],
+        properties: {
+          repoUrl: {
+            title: 'Repository Location',
+            description: `Accepts the format 'github.com?repo=reponame&owner=owner' where 'reponame' is the new repository name and 'owner' is an organization or username`,
+            type: 'string',
+          },
+          workflowId: {
+            title: 'Workflow ID',
+            description: 'The GitHub Action Workflow filename',
+            type: 'string',
+          },
+          branchOrTagName: {
+            title: 'Branch or Tag name',
+            description:
+              'The git branch or tag name used to dispatch the workflow',
+            type: 'string',
+          },
+        },
+      },
+    },
+    async handler(ctx) {
+      const { repoUrl, workflowId, branchOrTagName } = ctx.input;
+
+      const { owner, repo, host } = parseRepoUrl(repoUrl);
+
+      ctx.logger.info(
+        `Dispatching workflow ${workflowId} for repo ${repoUrl} on ${branchOrTagName}`,
+      );
+
+      const credentialsProvider = credentialsProviders.get(host);
+      const integrationConfig = integrations.github.byHost(host);
+
+      if (!credentialsProvider || !integrationConfig) {
+        throw new InputError(
+          `No matching integration configuration for host ${host}, please check your integrations config`,
+        );
+      }
+
+      const { token } = await credentialsProvider.getCredentials({
+        url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent(
+          repo,
+        )}`,
+      });
+
+      if (!token) {
+        throw new InputError(
+          `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`,
+        );
+      }
+
+      const client = new Octokit({
+        auth: token,
+        baseUrl: integrationConfig.config.apiBaseUrl,
+        previews: ['nebula-preview'],
+      });
+
+      await client.rest.actions.createWorkflowDispatch({
+        owner,
+        repo,
+        workflow_id: workflowId,
+        ref: branchOrTagName,
+      });
+
+      ctx.logger.info(`Workflow ${workflowId} dispatched successfully`);
+    },
+  });
+}

plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { createGithubActionsDispatchAction } from './githubActionsDispatch';

plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts

@@ -20,6 +20,7 @@ export * from './debug';
 export * from './fetch';
 export * from './filesystem';
 export * from './publish';
+export * from './github';
 
 export { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-backend-module-cookiecutter';
 export { runCommand } from './helpers';

plugins/search/api-report.md

@@ -143,6 +143,17 @@ export const SearchResult: ({
   children: (results: { results: SearchResult_2[] }) => JSX.Element;
 }) => JSX.Element;
 
+// Warning: (ae-forgotten-export) The symbol "SearchTypeProps" needs to be exported by the entry point index.d.ts
+// Warning: (ae-missing-release-tag) "SearchType" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export const SearchType: ({
+  values,
+  className,
+  name,
+  defaultValue,
+}: SearchTypeProps) => JSX.Element;
+
 // Warning: (ae-missing-release-tag) "SidebarSearch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/search/src/components/SearchType/SearchType.test.tsx

@@ -0,0 +1,231 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { screen, render, waitFor } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import { SearchType } from './SearchType';
+
+import { SearchContextProvider } from '../SearchContext';
+import { useApi } from '@backstage/core-plugin-api';
+
+jest.mock('@backstage/core-plugin-api', () => ({
+  ...jest.requireActual('@backstage/core-plugin-api'),
+  useApi: jest.fn().mockReturnValue({}),
+}));
+
+describe('SearchType', () => {
+  const initialState = {
+    term: '',
+    filters: {},
+    types: [],
+    pageCursor: '',
+  };
+
+  const name = 'field';
+  const values = ['value1', 'value2'];
+  const typeValues = ['preselected'];
+
+  const query = jest.fn().mockResolvedValue({});
+  (useApi as jest.Mock).mockReturnValue({ query: query });
+
+  afterAll(() => {
+    jest.resetAllMocks();
+  });
+
+  describe('Type Filter', () => {
+    it('Renders field name and values when provided as props', async () => {
+      render(
+        <SearchContextProvider initialState={initialState}>
+          <SearchType name={name} values={values} />
+        </SearchContextProvider>,
+      );
+
+      await waitFor(() => {
+        expect(screen.getByText(name)).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('button'));
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      expect(
+        screen.getByRole('option', { name: values[0] }),
+      ).toBeInTheDocument();
+      expect(
+        screen.getByRole('option', { name: values[1] }),
+      ).toBeInTheDocument();
+    });
+
+    it('Renders correctly based on type filter state', async () => {
+      render(
+        <SearchContextProvider
+          initialState={{
+            ...initialState,
+            types: [values[0]],
+          }}
+        >
+          <SearchType name={name} values={values} />
+        </SearchContextProvider>,
+      );
+
+      await waitFor(() => {
+        expect(screen.getByText(name)).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('button'));
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      expect(screen.getByRole('option', { name: values[0] })).toHaveAttribute(
+        'aria-selected',
+        'true',
+      );
+      expect(
+        screen.getByRole('option', { name: values[1] }),
+      ).not.toHaveAttribute('aria-selected');
+      expect(screen.getByRole('option', { name: 'All' })).not.toHaveAttribute(
+        'aria-selected',
+      );
+    });
+
+    it('Renders correctly based on type filter defaultValue', async () => {
+      render(
+        <SearchContextProvider initialState={initialState}>
+          <SearchType name={name} values={values} defaultValue={values[0]} />
+        </SearchContextProvider>,
+      );
+
+      await waitFor(() => {
+        expect(screen.getByText(name)).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('button'));
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      expect(screen.getByRole('option', { name: values[0] })).toHaveAttribute(
+        'aria-selected',
+        'true',
+      );
+      expect(
+        screen.getByRole('option', { name: values[1] }),
+      ).not.toHaveAttribute('aria-selected');
+      expect(screen.getByRole('option', { name: 'All' })).not.toHaveAttribute(
+        'aria-selected',
+      );
+    });
+
+    it('Selecting a value sets type filter state', async () => {
+      render(
+        <SearchContextProvider initialState={initialState}>
+          <SearchType name={name} values={values} />
+        </SearchContextProvider>,
+      );
+
+      await waitFor(() => {
+        expect(screen.getByText(name)).toBeInTheDocument();
+      });
+
+      const button = screen.getByRole('button');
+
+      userEvent.click(button);
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('option', { name: values[0] }));
+
+      await waitFor(() => {
+        expect(query).toHaveBeenLastCalledWith(
+          expect.objectContaining({
+            types: [values[0]],
+          }),
+        );
+      });
+
+      userEvent.click(button);
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('option', { name: 'All' }));
+
+      await waitFor(() => {
+        expect(query).toHaveBeenLastCalledWith(
+          expect.objectContaining({
+            types: [],
+          }),
+        );
+      });
+    });
+
+    it('Selecting a value maintains unrelated filter state, selecting All defaults to default empty state', async () => {
+      render(
+        <SearchContextProvider
+          initialState={{
+            ...initialState,
+            types: typeValues,
+          }}
+        >
+          <SearchType name={name} values={values} />
+        </SearchContextProvider>,
+      );
+
+      await waitFor(() => {
+        expect(screen.getByText(name)).toBeInTheDocument();
+      });
+
+      const button = screen.getByRole('button');
+
+      userEvent.click(button);
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('option', { name: values[0] }));
+
+      await waitFor(() => {
+        expect(query).toHaveBeenLastCalledWith(
+          expect.objectContaining({
+            types: [...typeValues, values[0]],
+          }),
+        );
+      });
+
+      userEvent.click(button);
+
+      await waitFor(() => {
+        expect(screen.getByRole('listbox')).toBeInTheDocument();
+      });
+
+      userEvent.click(screen.getByRole('option', { name: 'All' }));
+
+      await waitFor(() => {
+        expect(query).toHaveBeenLastCalledWith(expect.objectContaining([]));
+      });
+    });
+  });
+});

plugins/search/src/components/SearchType/SearchType.tsx

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { useSearch } from '../SearchContext';
+import { useEffectOnce } from 'react-use';
+import React, { ChangeEvent } from 'react';
+import {
+  Chip,
+  FormControl,
+  InputLabel,
+  makeStyles,
+  MenuItem,
+  Select,
+} from '@material-ui/core';
+
+const useStyles = makeStyles({
+  label: {
+    textTransform: 'capitalize',
+  },
+  chips: {
+    display: 'flex',
+    flexWrap: 'wrap',
+  },
+  chip: {
+    margin: 2,
+  },
+});
+
+export type SearchTypeProps = {
+  className?: string;
+  name: string;
+  values?: string[];
+  defaultValue?: string[] | string | null;
+};
+
+const SearchType = ({
+  values = [],
+  className,
+  name,
+  defaultValue,
+}: SearchTypeProps) => {
+  const classes = useStyles();
+  const { types, setTypes } = useSearch();
+
+  useEffectOnce(() => {
+    if (defaultValue && Array.isArray(defaultValue)) {
+      setTypes(defaultValue);
+    } else if (defaultValue) {
+      setTypes([defaultValue]);
+    }
+  });
+
+  const handleChange = (e: ChangeEvent<{ value: unknown }>) => {
+    const value = e.target.value as string[];
+    if (!value || value.includes('*')) {
+      setTypes([]);
+    } else {
+      setTypes(value.filter(it => it !== 'All'));
+    }
+  };
+
+  return (
+    <FormControl
+      className={className}
+      variant="filled"
+      fullWidth
+      data-testid="search-typefilter-next"
+    >
+      <InputLabel className={classes.label} margin="dense">
+        {name}
+      </InputLabel>
+      <Select
+        multiple
+        variant="outlined"
+        value={types.length ? types : ['All']}
+        onChange={handleChange}
+        renderValue={selected => (
+          <div className={classes.chips}>
+            {(selected as string[]).map(value => (
+              <Chip key={value} label={value} className={classes.chip} />
+            ))}
+          </div>
+        )}
+      >
+        <MenuItem value="*">
+          <em>All</em>
+        </MenuItem>
+        {values.map((value: string) => (
+          <MenuItem key={value} value={value}>
+            {value}
+          </MenuItem>
+        ))}
+      </Select>
+    </FormControl>
+  );
+};
+
+export { SearchType };

plugins/search/src/components/SearchType/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { SearchType } from './SearchType';

plugins/search/src/components/index.tsx

@@ -16,6 +16,7 @@
 
 export * from './Filters';
 export * from './SearchFilter';
+export * from './SearchType';
 export * from './SearchBar';
 export * from './SearchPage';
 export * from './SearchResult';

plugins/search/src/index.ts

@@ -32,6 +32,7 @@ export {
   useSearch,
   SearchPage as Router,
   SearchFilter,
+  SearchType,
   SearchFilterNext,
   SidebarSearch,
 } from './components';

plugins/techdocs-backend/api-report.md

@@ -3,9 +3,12 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+import { CatalogApi } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
+import { DocumentCollator } from '@backstage/search-common';
 import express from 'express';
 import { GeneratorBuilder } from '@backstage/techdocs-common';
+import { IndexableDocument } from '@backstage/search-common';
 import { Knex } from 'knex';
 import { Logger as Logger_2 } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -18,6 +21,54 @@ import { PublisherBase } from '@backstage/techdocs-common';
 // @public (undocumented)
 export function createRouter(options: RouterOptions): Promise<express.Router>;
 
+// Warning: (ae-missing-release-tag) "DefaultTechDocsCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class DefaultTechDocsCollator implements DocumentCollator {
+  constructor({
+    discovery,
+    locationTemplate,
+    logger,
+    catalogClient,
+    parallelismLimit,
+  }: {
+    discovery: PluginEndpointDiscovery;
+    logger: Logger_2;
+    locationTemplate?: string;
+    catalogClient?: CatalogApi;
+    parallelismLimit?: number;
+  });
+  // (undocumented)
+  protected applyArgsToFormat(
+    format: string,
+    args: Record<string, string>,
+  ): string;
+  // (undocumented)
+  protected discovery: PluginEndpointDiscovery;
+  // (undocumented)
+  execute(): Promise<TechDocsDocument[]>;
+  // (undocumented)
+  protected locationTemplate: string;
+  // (undocumented)
+  readonly type: string;
+}
+
+// Warning: (ae-missing-release-tag) "TechDocsDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface TechDocsDocument extends IndexableDocument {
+  // (undocumented)
+  kind: string;
+  // (undocumented)
+  lifecycle: string;
+  // (undocumented)
+  name: string;
+  // (undocumented)
+  namespace: string;
+  // (undocumented)
+  owner: string;
+}
+
 export * from '@backstage/techdocs-common';
 
 // (No @packageDocumentation comment for this package)

plugins/techdocs-backend/package.json

@@ -35,6 +35,8 @@
     "@backstage/catalog-model": "^0.9.0",
     "@backstage/config": "^0.1.5",
     "@backstage/errors": "^0.1.1",
+    "@backstage/integration": "^0.5.8",
+    "@backstage/search-common": "^0.1.2",
     "@backstage/techdocs-common": "^0.6.8",
     "@types/express": "^4.17.6",
     "cross-fetch": "^3.0.6",
@@ -43,10 +45,13 @@
     "express-promise-router": "^4.1.0",
     "fs-extra": "9.1.0",
     "knex": "^0.95.1",
+    "lodash": "^4.17.21",
+    "p-limit": "^3.1.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {
     "@backstage/cli": "^0.7.4",
+    "@backstage/test-utils": "^0.1.14",
     "@types/dockerode": "^3.2.1",
     "msw": "^0.29.0",
     "supertest": "^6.1.3"

plugins/techdocs-backend/src/DocsBuilder/builder.ts

@@ -20,6 +20,7 @@ import {
 } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { NotModifiedError } from '@backstage/errors';
+import { ScmIntegrationRegistry } from '@backstage/integration';
 import {
   GeneratorBase,
   GeneratorBuilder,
@@ -43,6 +44,7 @@ type DocsBuilderArguments = {
   entity: Entity;
   logger: Logger;
   config: Config;
+  scmIntegrations: ScmIntegrationRegistry;
   logStream?: Writable;
 };
 
@@ -53,6 +55,7 @@ export class DocsBuilder {
   private entity: Entity;
   private logger: Logger;
   private config: Config;
+  private scmIntegrations: ScmIntegrationRegistry;
   private logStream: Writable | undefined;
 
   constructor({
@@ -62,6 +65,7 @@ export class DocsBuilder {
     entity,
     logger,
     config,
+    scmIntegrations,
     logStream,
   }: DocsBuilderArguments) {
     this.preparer = preparers.get(entity);
@@ -70,6 +74,7 @@ export class DocsBuilder {
     this.entity = entity;
     this.logger = logger;
     this.config = config;
+    this.scmIntegrations = scmIntegrations;
     this.logStream = logStream;
   }
 
@@ -166,7 +171,10 @@ export class DocsBuilder {
       path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
     );
 
-    const parsedLocationAnnotation = getLocationForEntity(this.entity);
+    const parsedLocationAnnotation = getLocationForEntity(
+      this.entity,
+      this.scmIntegrations,
+    );
     await this.generator.run({
       inputDir: preparedDir,
       outputDir,

plugins/techdocs-backend/src/index.ts

@@ -15,4 +15,5 @@
  */
 
 export { createRouter } from './service/router';
+export * from './search';
 export * from '@backstage/techdocs-common';

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  PluginEndpointDiscovery,
+  getVoidLogger,
+} from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
+import { msw } from '@backstage/test-utils';
+import { setupServer } from 'msw/node';
+import { rest } from 'msw';
+
+const logger = getVoidLogger();
+
+const mockSearchDocIndex = {
+  config: {
+    lang: ['en'],
+    min_search_length: 3,
+    prebuild_index: false,
+    separator: '[\\s\\-]+',
+  },
+  docs: [
+    {
+      location: '',
+      text: 'docs docs docs',
+      title: 'Home',
+    },
+    {
+      location: 'local-development/',
+      text: 'Docs for first subtitle',
+      title: 'Local development',
+    },
+    {
+      location: 'local-development/#development',
+      text: 'Docs for sub-subtitle',
+      title: 'Development',
+    },
+  ],
+};
+
+const expectedEntities: Entity[] = [
+  {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Component',
+    metadata: {
+      name: 'test-entity-with-docs',
+      description: 'Documented description',
+      annotations: {
+        'backstage.io/techdocs-ref': './',
+      },
+    },
+    spec: {
+      type: 'dog',
+      lifecycle: 'experimental',
+      owner: 'someone',
+    },
+  },
+  {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Component',
+    metadata: {
+      name: 'test-entity',
+      description: 'The expected description',
+    },
+    spec: {
+      type: 'some-type',
+      lifecycle: 'experimental',
+    },
+  },
+];
+
+describe('DefaultTechDocsCollator', () => {
+  let mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery>;
+  let collator: DefaultTechDocsCollator;
+
+  const worker = setupServer();
+  msw.setupDefaultHandlers(worker);
+  beforeEach(() => {
+    mockDiscoveryApi = {
+      getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
+      getExternalBaseUrl: jest.fn(),
+    };
+    collator = new DefaultTechDocsCollator({
+      discovery: mockDiscoveryApi,
+      logger,
+    });
+
+    worker.use(
+      rest.get(
+        'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
+        (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
+      ),
+      rest.get('http://test-backend/entities', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json(expectedEntities)),
+      ),
+    );
+  });
+
+  it('fetches from the configured catalog and tech docs services', async () => {
+    const documents = await collator.execute();
+    expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
+    expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
+    expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
+  });
+
+  it('should create documents for each tech docs search index', async () => {
+    const documents = await collator.execute();
+    const entity = expectedEntities[0];
+    documents.forEach((document, idx) => {
+      expect(document).toMatchObject({
+        title: mockSearchDocIndex.docs[idx].title,
+        location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
+        text: mockSearchDocIndex.docs[idx].text,
+        namespace: 'default',
+        componentType: entity!.spec!.type,
+        lifecycle: entity!.spec!.lifecycle,
+        owner: '',
+      });
+    });
+  });
+
+  it('maps a returned entity with a custom locationTemplate', async () => {
+    // Provide an alternate location template.
+    collator = new DefaultTechDocsCollator({
+      discovery: mockDiscoveryApi,
+      locationTemplate: '/software/:name',
+      logger,
+    });
+
+    const documents = await collator.execute();
+    expect(documents[0]).toMatchObject({
+      location: '/software/test-entity-with-docs',
+    });
+  });
+});

plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model';
+import { IndexableDocument, DocumentCollator } from '@backstage/search-common';
+import fetch from 'cross-fetch';
+import unescape from 'lodash/unescape';
+import { Logger } from 'winston';
+import pLimit from 'p-limit';
+import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
+
+interface MkSearchIndexDoc {
+  title: string;
+  text: string;
+  location: string;
+}
+
+export interface TechDocsDocument extends IndexableDocument {
+  kind: string;
+  namespace: string;
+  name: string;
+  lifecycle: string;
+  owner: string;
+}
+
+export class DefaultTechDocsCollator implements DocumentCollator {
+  protected discovery: PluginEndpointDiscovery;
+  protected locationTemplate: string;
+  private readonly logger: Logger;
+  private readonly catalogClient: CatalogApi;
+  private readonly parallelismLimit: number;
+  public readonly type: string = 'techdocs';
+
+  constructor({
+    discovery,
+    locationTemplate,
+    logger,
+    catalogClient,
+    parallelismLimit = 10,
+  }: {
+    discovery: PluginEndpointDiscovery;
+    logger: Logger;
+    locationTemplate?: string;
+    catalogClient?: CatalogApi;
+    parallelismLimit?: number;
+  }) {
+    this.discovery = discovery;
+    this.locationTemplate =
+      locationTemplate || '/docs/:namespace/:kind/:name/:path';
+    this.logger = logger;
+    this.catalogClient =
+      catalogClient || new CatalogClient({ discoveryApi: discovery });
+    this.parallelismLimit = parallelismLimit;
+  }
+
+  async execute() {
+    const limit = pLimit(this.parallelismLimit);
+    const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
+    const entities = await this.catalogClient.getEntities({
+      fields: [
+        'kind',
+        'namespace',
+        'metadata.annotations',
+        'metadata.name',
+        'metadata.namespace',
+        'spec.type',
+        'spec.lifecycle',
+        'relations',
+      ],
+    });
+    const docPromises = entities.items
+      .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
+      .map((entity: Entity) =>
+        limit(
+          async (): Promise<TechDocsDocument[]> => {
+            const entityInfo = {
+              kind: entity.kind,
+              namespace: entity.metadata.namespace || 'default',
+              name: entity.metadata.name,
+            };
+
+            try {
+              const searchIndexResponse = await fetch(
+                DefaultTechDocsCollator.constructDocsIndexUrl(
+                  techDocsBaseUrl,
+                  entityInfo,
+                ),
+              );
+              const searchIndex = await searchIndexResponse.json();
+
+              return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+                title: unescape(doc.title),
+                text: unescape(doc.text || ''),
+                location: this.applyArgsToFormat(this.locationTemplate, {
+                  ...entityInfo,
+                  path: doc.location,
+                }),
+                ...entityInfo,
+                componentType: entity.spec?.type?.toString() || 'other',
+                lifecycle: (entity.spec?.lifecycle as string) || '',
+                owner:
+                  entity.relations?.find(r => r.type === RELATION_OWNED_BY)
+                    ?.target?.name || '',
+              }));
+            } catch (e) {
+              this.logger.warn(
+                `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
+                e,
+              );
+              return [];
+            }
+          },
+        ),
+      );
+    return (await Promise.all(docPromises)).flat();
+  }
+
+  protected applyArgsToFormat(
+    format: string,
+    args: Record<string, string>,
+  ): string {
+    let formatted = format;
+    for (const [key, value] of Object.entries(args)) {
+      formatted = formatted.replace(`:${key}`, value);
+    }
+    return formatted;
+  }
+
+  private static constructDocsIndexUrl(
+    techDocsBaseUrl: string,
+    entityInfo: { kind: string; namespace: string; name: string },
+  ) {
+    return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
+  }
+}

plugins/techdocs-backend/src/search/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
+export type { TechDocsDocument } from './DefaultTechDocsCollator';

plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts

@@ -19,6 +19,7 @@ import {
   PluginEndpointDiscovery,
 } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
 import {
   GeneratorBuilder,
   PreparerBuilder,
@@ -69,6 +70,7 @@ describe('DocsSynchronizer', () => {
       publisher,
       config: new ConfigReader({}),
       logger: getVoidLogger(),
+      scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
     });
   });
 

plugins/techdocs-backend/src/service/DocsSynchronizer.ts

@@ -17,6 +17,7 @@
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { NotFoundError } from '@backstage/errors';
+import { ScmIntegrationRegistry } from '@backstage/integration';
 import {
   GeneratorBuilder,
   PreparerBuilder,
@@ -36,19 +37,23 @@ export class DocsSynchronizer {
   private readonly publisher: PublisherBase;
   private readonly logger: winston.Logger;
   private readonly config: Config;
+  private readonly scmIntegrations: ScmIntegrationRegistry;
 
   constructor({
     publisher,
     logger,
     config,
+    scmIntegrations,
   }: {
     publisher: PublisherBase;
     logger: winston.Logger;
     config: Config;
+    scmIntegrations: ScmIntegrationRegistry;
   }) {
     this.config = config;
     this.logger = logger;
     this.publisher = publisher;
+    this.scmIntegrations = scmIntegrations;
   }
 
   async doSync({
@@ -87,19 +92,20 @@ export class DocsSynchronizer {
       return;
     }
 
-    const docsBuilder = new DocsBuilder({
-      preparers,
-      generators,
-      publisher: this.publisher,
-      logger: taskLogger,
-      entity,
-      config: this.config,
-      logStream,
-    });
-
     let foundDocs = false;
 
     try {
+      const docsBuilder = new DocsBuilder({
+        preparers,
+        generators,
+        publisher: this.publisher,
+        logger: taskLogger,
+        entity,
+        config: this.config,
+        scmIntegrations: this.scmIntegrations,
+        logStream,
+      });
+
       const updated = await docsBuilder.build();
 
       if (!updated) {

plugins/techdocs-backend/src/service/router.ts

@@ -29,6 +29,7 @@ import express, { Response } from 'express';
 import Router from 'express-promise-router';
 import { Knex } from 'knex';
 import { Logger } from 'winston';
+import { ScmIntegrations } from '@backstage/integration';
 import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
 
 /**
@@ -79,10 +80,12 @@ export async function createRouter(
   const router = Router();
   const { publisher, config, logger, discovery } = options;
   const catalogClient = new CatalogClient({ discoveryApi: discovery });
+  const scmIntegrations = ScmIntegrations.fromConfig(config);
   const docsSynchronizer = new DocsSynchronizer({
-    publisher: publisher,
-    logger: logger,
-    config: config,
+    publisher,
+    logger,
+    config,
+    scmIntegrations,
   });
 
   router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
@@ -126,7 +129,7 @@ export async function createRouter(
         )
       ).json()) as Entity;
 
-      const locationMetadata = getLocationForEntity(entity);
+      const locationMetadata = getLocationForEntity(entity, scmIntegrations);
       res.json({ ...entity, locationMetadata });
     } catch (err) {
       logger.info(

plugins/techdocs/api-report.md

@@ -25,6 +25,17 @@ export const DocsCardGrid: ({
   entities: Entity[] | undefined;
 }) => JSX.Element | null;
 
+// Warning: (ae-missing-release-tag) "DocsResultListItem" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export const DocsResultListItem: ({
+  result,
+  lineClamp,
+}: {
+  result: any;
+  lineClamp?: number | undefined;
+}) => JSX.Element;
+
 // Warning: (ae-missing-release-tag) "DocsTable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/techdocs/package.json

@@ -51,6 +51,7 @@
     "react-router": "6.0.0-beta.0",
     "react-router-dom": "6.0.0-beta.0",
     "react-use": "^17.2.4",
+    "react-text-truncate": "^0.16.0",
     "sanitize-html": "^2.3.2"
   },
   "devDependencies": {

plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.test.tsx

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+import { DocsResultListItem } from './DocsResultListItem';
+
+// Using canvas to render text..
+jest.mock('react-text-truncate', () => {
+  return ({ text }: { text: string }) => <span>{text}</span>;
+});
+
+const validResult = {
+  location: 'https://backstage.io/docs',
+  title: 'Documentation',
+  text:
+    'Backstage is an open-source developer portal that puts the developer experience first.',
+  kind: 'library',
+  namespace: '',
+  name: 'Backstage',
+  lifecycle: 'production',
+};
+
+describe('DocsResultListItem test', () => {
+  it('should render search doc passed in', async () => {
+    const { findByText } = render(<DocsResultListItem result={validResult} />);
+
+    expect(
+      await findByText('Documentation | Backstage docs'),
+    ).toBeInTheDocument();
+    expect(
+      await findByText(
+        'Backstage is an open-source developer portal that puts the developer experience first.',
+      ),
+    ).toBeInTheDocument();
+  });
+});

plugins/techdocs/src/components/DocsResultListItem/DocsResultListItem.tsx

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { Divider, ListItem, ListItemText, makeStyles } from '@material-ui/core';
+import { Link } from '@backstage/core-components';
+import TextTruncate from 'react-text-truncate';
+
+const useStyles = makeStyles({
+  flexContainer: {
+    flexWrap: 'wrap',
+  },
+  itemText: {
+    width: '100%',
+    marginBottom: '1rem',
+  },
+});
+
+export const DocsResultListItem = ({
+  result,
+  lineClamp = 5,
+}: {
+  result: any;
+  lineClamp?: number;
+}) => {
+  const classes = useStyles();
+  return (
+    <Link to={result.location}>
+      <ListItem alignItems="flex-start" className={classes.flexContainer}>
+        <ListItemText
+          className={classes.itemText}
+          primaryTypographyProps={{ variant: 'h6' }}
+          primary={`${result.title} | ${result.name} docs `}
+          secondary={
+            <TextTruncate
+              line={lineClamp}
+              truncateText="…"
+              text={result.text}
+              element="span"
+            />
+          }
+        />
+      </ListItem>
+      <Divider component="li" />
+    </Link>
+  );
+};

plugins/techdocs/src/components/DocsResultListItem/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { DocsResultListItem } from './DocsResultListItem';

plugins/techdocs/src/index.ts

@@ -19,6 +19,7 @@ export { techdocsApiRef, techdocsStorageApiRef } from './api';
 export type { TechDocsApi, TechDocsStorageApi } from './api';
 export { TechDocsClient, TechDocsStorageClient } from './client';
 export type { PanelType } from './home/components/TechDocsCustomHome';
+export * from './components/DocsResultListItem';
 export {
   DocsCardGrid,
   DocsTable,

plugins/techdocs/src/reader/components/Reader.tsx

@@ -61,18 +61,18 @@ const useStyles = makeStyles<BackstageTheme>(() => ({
 
 export const Reader = ({ entityId, onReady }: Props) => {
   const { kind, namespace, name } = entityId;
-  const { '*': path } = useParams();
   const theme = useTheme<BackstageTheme>();
   const classes = useStyles();
 
   const {
     state,
+    path,
     contentReload,
     content: rawPage,
     contentErrorMessage,
     syncErrorMessage,
     buildLog,
-  } = useReaderState(kind, namespace, name, path);
+  } = useReaderState(kind, namespace, name, useParams()['*']);
 
   const techdocsStorageApi = useApi(techdocsStorageApiRef);
   const [sidebars, setSidebars] = useState<HTMLElement[]>();
@@ -109,31 +109,26 @@ export const Reader = ({ entityId, onReady }: Props) => {
     // an update to "state" might lead to an updated UI so we include it as a trigger
   }, [updateSidebarPosition, state]);
 
-  useEffect(() => {
-    if (!rawPage || !shadowDomRef.current) {
-      return;
-    }
-    if (onReady) {
-      onReady();
-    }
-    // Pre-render
-    const transformedElement = transformer(rawPage, [
-      sanitizeDOM(),
-      addBaseUrl({
-        techdocsStorageApi,
-        entityId: {
-          kind,
-          name,
-          namespace,
-        },
-        path,
-      }),
-      rewriteDocLinks(),
-      removeMkdocsHeader(),
-      simplifyMkdocsFooter(),
-      addGitFeedbackLink(scmIntegrationsApi),
-      injectCss({
-        css: `
+  // a function that performs transformations that are executed prior to adding it to the DOM
+  const preRender = useCallback(
+    (rawContent: string, contentPath: string) =>
+      transformer(rawContent, [
+        sanitizeDOM(),
+        addBaseUrl({
+          techdocsStorageApi,
+          entityId: {
+            kind,
+            name,
+            namespace,
+          },
+          path: contentPath,
+        }),
+        rewriteDocLinks(),
+        removeMkdocsHeader(),
+        simplifyMkdocsFooter(),
+        addGitFeedbackLink(scmIntegrationsApi),
+        injectCss({
+          css: `
         body {
           font-family: ${theme.typography.fontFamily};
           --md-text-color: ${theme.palette.text.primary};
@@ -190,21 +185,21 @@ export const Reader = ({ entityId, onReady }: Props) => {
           }
         }
       `,
-      }),
-      injectCss({
-        // Disable CSS animations on link colors as they lead to issues in dark
-        // mode. The dark mode color theme is applied later and theirfore there
-        // is always an animation from light to dark mode when navigation
-        // between pages.
-        css: `
+        }),
+        injectCss({
+          // Disable CSS animations on link colors as they lead to issues in dark
+          // mode. The dark mode color theme is applied later and theirfore there
+          // is always an animation from light to dark mode when navigation
+          // between pages.
+          css: `
         .md-nav__link, .md-typeset a, .md-typeset a::before, .md-typeset .headerlink {
           transition: none;
         }
         `,
-      }),
-      injectCss({
-        // Properly style code blocks.
-        css: `
+        }),
+        injectCss({
+          // Properly style code blocks.
+          css: `
         .md-typeset pre > code::-webkit-scrollbar-thumb {
           background-color: hsla(0, 0%, 0%, 0.32);
         }
@@ -212,17 +207,17 @@ export const Reader = ({ entityId, onReady }: Props) => {
           background-color: hsla(0, 0%, 0%, 0.87);
         }
         `,
-      }),
-      injectCss({
-        // Admonitions and others are using SVG masks to define icons. These
-        // masks are defined as CSS variables.
-        // As the MkDocs output is rendered in shadow DOM, the CSS variable
-        // definitions on the root selector are not applied. Instead, the have
-        // to be applied on :host.
-        // As there is no way to transform the served main*.css yet (for
-        // example in the backend), we have to copy from main*.css and modify
-        // them.
-        css: `
+        }),
+        injectCss({
+          // Admonitions and others are using SVG masks to define icons. These
+          // masks are defined as CSS variables.
+          // As the MkDocs output is rendered in shadow DOM, the CSS variable
+          // definitions on the root selector are not applied. Instead, the have
+          // to be applied on :host.
+          // As there is no way to transform the served main*.css yet (for
+          // example in the backend), we have to copy from main*.css and modify
+          // them.
+          css: `
         :host {
           --md-admonition-icon--note: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z"/></svg>');
           --md-admonition-icon--abstract: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 5h16v2H4V5m0 4h16v2H4V9m0 4h16v2H4v-2m0 4h10v2H4v-2z"/></svg>');
@@ -248,97 +243,129 @@ export const Reader = ({ entityId, onReady }: Props) => {
           --md-tasklist-icon--checked: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2C6.5 2 2 6.5 2 12s4.5 10 10 10 10-4.5 10-10S17.5 2 12 2m-2 15l-5-5 1.41-1.41L10 14.17l7.59-7.59L19 8l-9 9z"/></svg>');
         }
         `,
-      }),
-    ]);
+        }),
+      ]),
+    [
+      kind,
+      name,
+      namespace,
+      scmIntegrationsApi,
+      techdocsStorageApi,
+      theme.palette.background.default,
+      theme.palette.background.paper,
+      theme.palette.primary.main,
+      theme.palette.text.primary,
+      theme.typography.fontFamily,
+    ],
+  );
 
-    if (!transformedElement) {
-      return; // An unexpected error occurred
+  // a function that performs transformations that are executed after adding it to the DOM
+  const postRender = useCallback(
+    async (shadowRoot: ShadowRoot) =>
+      transformer(shadowRoot.children[0], [
+        dom => {
+          setTimeout(() => {
+            // Scoll to the desired anchor on initial navigation
+            if (window.location.hash) {
+              const hash = window.location.hash.slice(1);
+              shadowRoot?.getElementById(hash)?.scrollIntoView();
+            }
+          }, 200);
+          return dom;
+        },
+        addLinkClickListener({
+          baseUrl: window.location.origin,
+          onClick: (_: MouseEvent, url: string) => {
+            const parsedUrl = new URL(url);
+
+            if (parsedUrl.hash) {
+              navigate(`${parsedUrl.pathname}${parsedUrl.hash}`);
+
+              // Scroll to hash if it's on the current page
+              shadowRoot
+                ?.getElementById(parsedUrl.hash.slice(1))
+                ?.scrollIntoView();
+            } else {
+              navigate(parsedUrl.pathname);
+            }
+          },
+        }),
+        onCssReady({
+          docStorageUrl: await techdocsStorageApi.getApiOrigin(),
+          onLoading: (dom: Element) => {
+            (dom as HTMLElement).style.setProperty('opacity', '0');
+          },
+          onLoaded: (dom: Element) => {
+            (dom as HTMLElement).style.removeProperty('opacity');
+            // disable MkDocs drawer toggling ('for' attribute => checkbox mechanism)
+            (dom as HTMLElement)
+              .querySelector('.md-nav__title')
+              ?.removeAttribute('for');
+            const sideDivs: HTMLElement[] = Array.from(
+              shadowRoot!.querySelectorAll('.md-sidebar'),
+            );
+            setSidebars(sideDivs);
+            // set sidebar height so they don't initially render in wrong position
+            const docTopPosition = (dom as HTMLElement).getBoundingClientRect()
+              .top;
+            const mdTabs = dom.querySelector('.md-container > .md-tabs');
+            sideDivs!.forEach(sidebar => {
+              sidebar.style.top = mdTabs
+                ? `${docTopPosition + mdTabs.getBoundingClientRect().height}px`
+                : `${docTopPosition}px`;
+            });
+          },
+        }),
+      ]),
+    [navigate, techdocsStorageApi],
+  );
+
+  useEffect(() => {
+    if (!rawPage || !shadowDomRef.current) {
+      // clear the shadow dom if no content is available
+      if (shadowDomRef.current?.shadowRoot) {
+        shadowDomRef.current.shadowRoot.innerHTML = '';
+      }
+      return () => {};
+    }
+    if (onReady) {
+      onReady();
     }
 
-    const shadowDiv: HTMLElement = shadowDomRef.current!;
-    const shadowRoot =
-      shadowDiv.shadowRoot || shadowDiv.attachShadow({ mode: 'open' });
-    Array.from(shadowRoot.children).forEach(child =>
-      shadowRoot.removeChild(child),
-    );
-    shadowRoot.appendChild(transformedElement);
+    // if false, there is already a newer execution of this effect
+    let shouldReplaceContent = true;
 
-    // Scroll to top after render
-    window.scroll({ top: 0 });
+    // Pre-render
+    preRender(rawPage, path).then(async transformedElement => {
+      if (!transformedElement?.innerHTML) {
+        return; // An unexpected error occurred
+      }
 
-    // Post-render
-    transformer(shadowRoot.children[0], [
-      dom => {
-        setTimeout(() => {
-          // Scoll to the desired anchor on initial navigation
-          if (window.location.hash) {
-            const hash = window.location.hash.slice(1);
-            shadowRoot?.getElementById(hash)?.scrollIntoView();
-          }
-        }, 200);
-        return dom;
-      },
-      addLinkClickListener({
-        baseUrl: window.location.origin,
-        onClick: (_: MouseEvent, url: string) => {
-          const parsedUrl = new URL(url);
+      // don't manipulate the shadow dom if this isn't the latest effect execution
+      if (!shouldReplaceContent) {
+        return;
+      }
 
-          if (parsedUrl.hash) {
-            navigate(`${parsedUrl.pathname}${parsedUrl.hash}`);
+      const shadowDiv: HTMLElement = shadowDomRef.current!;
+      const shadowRoot =
+        shadowDiv.shadowRoot || shadowDiv.attachShadow({ mode: 'open' });
+      Array.from(shadowRoot.children).forEach(child =>
+        shadowRoot.removeChild(child),
+      );
+      shadowRoot.appendChild(transformedElement);
 
-            // Scroll to hash if it's on the current page
-            shadowRoot
-              ?.getElementById(parsedUrl.hash.slice(1))
-              ?.scrollIntoView();
-          } else {
-            navigate(parsedUrl.pathname);
-          }
-        },
-      }),
-      onCssReady({
-        docStorageUrl: techdocsStorageApi.getApiOrigin(),
-        onLoading: (dom: Element) => {
-          (dom as HTMLElement).style.setProperty('opacity', '0');
-        },
-        onLoaded: (dom: Element) => {
-          (dom as HTMLElement).style.removeProperty('opacity');
-          // disable MkDocs drawer toggling ('for' attribute => checkbox mechanism)
-          (dom as HTMLElement)
-            .querySelector('.md-nav__title')
-            ?.removeAttribute('for');
-          const sideDivs: HTMLElement[] = Array.from(
-            shadowRoot!.querySelectorAll('.md-sidebar'),
-          );
-          setSidebars(sideDivs);
-          // set sidebar height so they don't initially render in wrong position
-          const docTopPosition = (dom as HTMLElement).getBoundingClientRect()
-            .top;
-          const mdTabs = dom.querySelector('.md-container > .md-tabs');
-          sideDivs!.forEach(sidebar => {
-            sidebar.style.top = mdTabs
-              ? `${docTopPosition + mdTabs.getBoundingClientRect().height}px`
-              : `${docTopPosition}px`;
-          });
-        },
-      }),
-    ]);
-  }, [
-    path,
-    kind,
-    namespace,
-    name,
-    rawPage,
-    navigate,
-    onReady,
-    shadowDomRef,
-    techdocsStorageApi,
-    theme.typography.fontFamily,
-    theme.palette.text.primary,
-    theme.palette.primary.main,
-    theme.palette.background.paper,
-    theme.palette.background.default,
-    scmIntegrationsApi,
-  ]);
+      // Scroll to top after render
+      window.scroll({ top: 0 });
+
+      // Post-render
+      await postRender(shadowRoot);
+    });
+
+    // cancel this execution
+    return () => {
+      shouldReplaceContent = false;
+    };
+  }, [onReady, path, postRender, preRender, rawPage]);
 
   return (
     <>

plugins/techdocs/src/reader/components/useReaderState.test.tsx

@@ -86,7 +86,7 @@ describe('useReaderState', () => {
     };
 
     it('should return a copy of the state', () => {
-      expect(reducer(oldState, { type: 'navigate', path: '/' })).toEqual({
+      expect(reducer(oldState, { type: 'content', path: '/' })).toEqual({
         activeSyncState: 'CHECKING',
         contentLoading: false,
         path: '/',
@@ -102,13 +102,13 @@ describe('useReaderState', () => {
     });
 
     it.each`
-      type          | oldActiveSyncState      | newActiveSyncState
-      ${'content'}  | ${'BUILD_READY'}        | ${'UP_TO_DATE'}
-      ${'content'}  | ${'BUILD_READY_RELOAD'} | ${'UP_TO_DATE'}
-      ${'navigate'} | ${'BUILD_READY'}        | ${'UP_TO_DATE'}
-      ${'navigate'} | ${'BUILD_READY_RELOAD'} | ${'UP_TO_DATE'}
-      ${'sync'}     | ${'BUILD_READY'}        | ${undefined}
-      ${'sync'}     | ${'BUILD_READY_RELOAD'} | ${undefined}
+      type                | oldActiveSyncState      | newActiveSyncState
+      ${'contentLoading'} | ${'BUILD_READY'}        | ${'UP_TO_DATE'}
+      ${'contentLoading'} | ${'BUILD_READY_RELOAD'} | ${'UP_TO_DATE'}
+      ${'content'}        | ${'BUILD_READY'}        | ${'UP_TO_DATE'}
+      ${'content'}        | ${'BUILD_READY_RELOAD'} | ${'UP_TO_DATE'}
+      ${'sync'}           | ${'BUILD_READY'}        | ${undefined /* undefined, because we don't set an input */}
+      ${'sync'}           | ${'BUILD_READY_RELOAD'} | ${undefined /* undefined, because we don't set an input */}
     `(
       'should, when type=$type and activeSyncState=$oldActiveSyncState, set activeSyncState=$newActiveSyncState',
       ({ type, oldActiveSyncState, newActiveSyncState }) => {
@@ -124,18 +124,45 @@ describe('useReaderState', () => {
       },
     );
 
-    describe('"content" action', () => {
+    describe('"contentLoading" action', () => {
       it('should set loading', () => {
+        expect(
+          reducer(oldState, {
+            type: 'contentLoading',
+          }),
+        ).toEqual({
+          ...oldState,
+          contentLoading: true,
+        });
+      });
+
+      it('should keep content', () => {
         expect(
           reducer(
             {
               ...oldState,
               content: 'some-old-content',
+            },
+            {
+              type: 'contentLoading',
+            },
+          ),
+        ).toEqual({
+          ...oldState,
+          contentLoading: true,
+          content: 'some-old-content',
+        });
+      });
+
+      it('should reset errors', () => {
+        expect(
+          reducer(
+            {
+              ...oldState,
               contentError: new Error(),
             },
             {
-              type: 'content',
-              contentLoading: true,
+              type: 'contentLoading',
             },
           ),
         ).toEqual({
@@ -143,7 +170,9 @@ describe('useReaderState', () => {
           contentLoading: true,
         });
       });
+    });
 
+    describe('"content" action', () => {
       it('should set content', () => {
         expect(
           reducer(
@@ -164,6 +193,27 @@ describe('useReaderState', () => {
         });
       });
 
+      it('should set content and update path', () => {
+        expect(
+          reducer(
+            {
+              ...oldState,
+              contentLoading: true,
+            },
+            {
+              type: 'content',
+              content: 'asdf',
+              path: '/new-path',
+            },
+          ),
+        ).toEqual({
+          ...oldState,
+          contentLoading: false,
+          content: 'asdf',
+          path: '/new-path',
+        });
+      });
+
       it('should set error', () => {
         expect(
           reducer(
@@ -185,20 +235,6 @@ describe('useReaderState', () => {
       });
     });
 
-    describe('"navigate" action', () => {
-      it('should work', () => {
-        expect(
-          reducer(oldState, {
-            type: 'navigate',
-            path: '/',
-          }),
-        ).toEqual({
-          ...oldState,
-          path: '/',
-        });
-      });
-    });
-
     describe('"sync" action', () => {
       it('should update state', () => {
         expect(
@@ -256,6 +292,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CHECKING',
+          path: '/example',
           content: undefined,
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -267,6 +304,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CONTENT_FRESH',
+          path: '/example',
           content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -313,6 +351,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CHECKING',
+          path: '/example',
           content: undefined,
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -324,6 +363,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'INITIAL_BUILD',
+          path: '/example',
           content: undefined,
           contentErrorMessage: 'NotFoundError: Page Not Found',
           syncErrorMessage: undefined,
@@ -335,6 +375,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CHECKING',
+          path: '/example',
           content: undefined,
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -346,6 +387,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CONTENT_FRESH',
+          path: '/example',
           content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -394,6 +436,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CHECKING',
+          path: '/example',
           content: undefined,
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -405,6 +448,7 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CONTENT_FRESH',
+          path: '/example',
           content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -416,6 +460,7 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CONTENT_STALE_REFRESHING',
+          path: '/example',
           content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -427,6 +472,7 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CONTENT_STALE_READY',
+          path: '/example',
           content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -441,7 +487,8 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CHECKING',
-          content: undefined,
+          path: '/example',
+          content: 'my content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
           buildLog: [],
@@ -452,6 +499,7 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CONTENT_FRESH',
+          path: '/example',
           content: 'my new content',
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -475,6 +523,103 @@ describe('useReaderState', () => {
       });
     });
 
+    it('should handle navigation', async () => {
+      techdocsStorageApi.getEntityDocs
+        .mockResolvedValueOnce('my content')
+        .mockImplementationOnce(async () => {
+          await new Promise(resolve => setTimeout(resolve, 1100));
+          return 'my new content';
+        })
+        .mockRejectedValueOnce(new NotFoundError('Some error description'));
+      techdocsStorageApi.syncEntityDocs.mockResolvedValue('cached');
+
+      await act(async () => {
+        const { result, waitForValueToChange, rerender } = await renderHook(
+          ({ path }: { path: string }) =>
+            useReaderState('Component', 'default', 'backstage', path),
+          { initialProps: { path: '/example' }, wrapper: Wrapper as any },
+        );
+
+        expect(result.current).toEqual({
+          state: 'CHECKING',
+          path: '/example',
+          content: undefined,
+          contentErrorMessage: undefined,
+          syncErrorMessage: undefined,
+          buildLog: [],
+          contentReload: expect.any(Function),
+        });
+
+        // show the content
+        await waitForValueToChange(() => result.current.state);
+        expect(result.current).toEqual({
+          state: 'CONTENT_FRESH',
+          path: '/example',
+          content: 'my content',
+          contentErrorMessage: undefined,
+          syncErrorMessage: undefined,
+          buildLog: [],
+          contentReload: expect.any(Function),
+        });
+
+        // navigate
+        rerender({ path: '/new' });
+
+        await waitForValueToChange(() => result.current.state);
+        expect(result.current).toEqual({
+          state: 'CHECKING',
+          path: '/example',
+          content: 'my content',
+          contentErrorMessage: undefined,
+          syncErrorMessage: undefined,
+          buildLog: [],
+          contentReload: expect.any(Function),
+        });
+
+        await waitForValueToChange(() => result.current.state);
+        expect(result.current).toEqual({
+          state: 'CONTENT_FRESH',
+          path: '/new',
+          content: 'my new content',
+          contentErrorMessage: undefined,
+          syncErrorMessage: undefined,
+          buildLog: [],
+          contentReload: expect.any(Function),
+        });
+
+        // navigate
+        rerender({ path: '/missing' });
+
+        await waitForValueToChange(() => result.current.state);
+        expect(result.current).toEqual({
+          state: 'CONTENT_NOT_FOUND',
+          path: '/missing',
+          content: undefined,
+          contentErrorMessage: 'NotFoundError: Some error description',
+          syncErrorMessage: undefined,
+          buildLog: [],
+          contentReload: expect.any(Function),
+        });
+
+        expect(techdocsStorageApi.getEntityDocs).toBeCalledWith(
+          { kind: 'Component', namespace: 'default', name: 'backstage' },
+          '/example',
+        );
+        expect(techdocsStorageApi.getEntityDocs).toBeCalledWith(
+          { kind: 'Component', namespace: 'default', name: 'backstage' },
+          '/new',
+        );
+        expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith(
+          {
+            kind: 'Component',
+            namespace: 'default',
+            name: 'backstage',
+          },
+          expect.any(Function),
+        );
+      });
+    });
+
     it('should handle content error', async () => {
       techdocsStorageApi.getEntityDocs.mockRejectedValue(
         new NotFoundError('Some error description'),
@@ -489,6 +634,7 @@ describe('useReaderState', () => {
 
         expect(result.current).toEqual({
           state: 'CHECKING',
+          path: '/example',
           content: undefined,
           contentErrorMessage: undefined,
           syncErrorMessage: undefined,
@@ -500,6 +646,7 @@ describe('useReaderState', () => {
         await waitForValueToChange(() => result.current.state);
         expect(result.current).toEqual({
           state: 'CONTENT_NOT_FOUND',
+          path: '/example',
           content: undefined,
           contentErrorMessage: 'NotFoundError: Some error description',
           syncErrorMessage: undefined,

plugins/techdocs/src/reader/components/useReaderState.ts

@@ -15,7 +15,7 @@
  */
 
 import { useApi } from '@backstage/core-plugin-api';
-import { useEffect, useMemo, useReducer, useRef } from 'react';
+import { useMemo, useReducer, useRef } from 'react';
 import { useAsync, useAsyncRetry } from 'react-use';
 import { techdocsStorageApiRef } from '../../api';
 
@@ -131,13 +131,13 @@ type ReducerActions =
       state: SyncStates;
       syncError?: Error;
     }
+  | { type: 'contentLoading' }
   | {
       type: 'content';
+      path?: string;
       content?: string;
-      contentLoading?: true;
       contentError?: Error;
     }
-  | { type: 'navigate'; path: string }
   | { type: 'buildLog'; log: string };
 
 type ReducerState = {
@@ -186,14 +186,22 @@ export function reducer(
       newState.syncError = action.syncError;
       break;
 
-    case 'content':
-      newState.content = action.content;
-      newState.contentLoading = action.contentLoading ?? false;
-      newState.contentError = action.contentError;
+    case 'contentLoading':
+      newState.contentLoading = true;
+
+      // only reset errors but keep the old content until it is replaced by the 'content' action
+      newState.contentError = undefined;
       break;
 
-    case 'navigate':
-      newState.path = action.path;
+    case 'content':
+      // only override the path if it is part of the action
+      if (typeof action.path === 'string') {
+        newState.path = action.path;
+      }
+
+      newState.contentLoading = false;
+      newState.content = action.content;
+      newState.contentError = action.contentError;
       break;
 
     case 'buildLog':
@@ -204,10 +212,10 @@ export function reducer(
       throw new Error();
   }
 
-  // a navigation or a content update loads fresh content so the build is updated to being up-to-date
+  // a content update loads fresh content so the build is updated to being up-to-date
   if (
     ['BUILD_READY', 'BUILD_READY_RELOAD'].includes(newState.activeSyncState) &&
-    ['content', 'navigate'].includes(action.type)
+    ['contentLoading', 'content'].includes(action.type)
   ) {
     newState.activeSyncState = 'UP_TO_DATE';
     newState.buildLog = [];
@@ -223,6 +231,7 @@ export function useReaderState(
   path: string,
 ): {
   state: ContentStateTypes;
+  path: string;
   contentReload: () => void;
   content?: string;
   contentErrorMessage?: string;
@@ -238,14 +247,9 @@ export function useReaderState(
 
   const techdocsStorageApi = useApi(techdocsStorageApiRef);
 
-  // convert all path changes into actions
-  useEffect(() => {
-    dispatch({ type: 'navigate', path });
-  }, [path]);
-
   // try to load the content. the function will fire events and we don't care for the return values
   const { retry: contentReload } = useAsyncRetry(async () => {
-    dispatch({ type: 'content', contentLoading: true });
+    dispatch({ type: 'contentLoading' });
 
     try {
       const entityDocs = await techdocsStorageApi.getEntityDocs(
@@ -253,11 +257,12 @@ export function useReaderState(
         path,
       );
 
-      dispatch({ type: 'content', content: entityDocs });
+      // update content and path at the same time
+      dispatch({ type: 'content', content: entityDocs, path });
 
       return entityDocs;
     } catch (e) {
-      dispatch({ type: 'content', contentError: e });
+      dispatch({ type: 'content', contentError: e, path });
     }
 
     return undefined;
@@ -335,6 +340,7 @@ export function useReaderState(
   return {
     state: displayState,
     contentReload,
+    path: state.path,
     content: state.content,
     contentErrorMessage: state.contentError?.toString(),
     syncErrorMessage: state.syncError?.toString(),

plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts

@@ -15,9 +15,9 @@
  */
 
 import { waitFor } from '@testing-library/react';
-import { createTestShadowDom } from '../../test-utils';
-import { addBaseUrl } from '../transformers';
 import { TechDocsStorageApi } from '../../api';
+import { createTestShadowDom } from '../../test-utils';
+import { addBaseUrl } from './addBaseUrl';
 
 const DOC_STORAGE_URL = 'https://example-host.storage.googleapis.com';
 const API_ORIGIN_URL = 'https://backstage.example.com/api/techdocs';
@@ -62,8 +62,8 @@ describe('addBaseUrl', () => {
     global.fetch = originalFetch;
   });
 
-  it('contains relative paths', () => {
-    createTestShadowDom(fixture, {
+  it('contains relative paths', async () => {
+    await createTestShadowDom(fixture, {
       preTransformers: [
         addBaseUrl({
           techdocsStorageApi,
@@ -110,7 +110,7 @@ describe('addBaseUrl', () => {
       text: jest.fn().mockResolvedValue(svgContent),
     });
 
-    const root = createTestShadowDom('<img id="x" src="test.svg" />', {
+    const root = await createTestShadowDom('<img id="x" src="test.svg" />', {
       preTransformers: [
         addBaseUrl({
           techdocsStorageApi,
@@ -137,7 +137,7 @@ describe('addBaseUrl', () => {
       text: jest.fn().mockResolvedValue(svgContent),
     });
 
-    const root = createTestShadowDom(
+    const root = await createTestShadowDom(
       `<img id="x" src="${API_ORIGIN_URL}/test.svg" />`,
       {
         preTransformers: [
@@ -162,16 +162,19 @@ describe('addBaseUrl', () => {
 
   it('does not inline external svgs', async () => {
     const expectedSrc = 'https://example.com/test.svg';
-    const root = createTestShadowDom(`<img id="x" src="${expectedSrc}" />`, {
-      preTransformers: [
-        addBaseUrl({
-          techdocsStorageApi,
-          entityId: mockEntityId,
-          path: '',
-        }),
-      ],
-      postTransformers: [],
-    });
+    const root = await createTestShadowDom(
+      `<img id="x" src="${expectedSrc}" />`,
+      {
+        preTransformers: [
+          addBaseUrl({
+            techdocsStorageApi,
+            entityId: mockEntityId,
+            path: '',
+          }),
+        ],
+        postTransformers: [],
+      },
+    );
 
     await new Promise<void>(done => {
       process.nextTick(() => {

plugins/techdocs/src/reader/transformers/addBaseUrl.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 import { EntityName } from '@backstage/catalog-model';
-import type { Transformer } from './transformer';
 import { TechDocsStorageApi } from '../../api';
+import type { Transformer } from './transformer';
 
 type AddBaseUrlOptions = {
   techdocsStorageApi: TechDocsStorageApi;
@@ -44,14 +44,15 @@ export const addBaseUrl = ({
   entityId,
   path,
 }: AddBaseUrlOptions): Transformer => {
-  return dom => {
-    const updateDom = <T extends Element>(
+  return async dom => {
+    const apiOrigin = await techdocsStorageApi.getApiOrigin();
+
+    const updateDom = async <T extends Element>(
       list: HTMLCollectionOf<T> | NodeListOf<T>,
       attributeName: string,
-    ): void => {
-      Array.from(list)
-        .filter(elem => !!elem.getAttribute(attributeName))
-        .forEach(async (elem: T) => {
+    ) => {
+      for (const elem of list) {
+        if (elem.hasAttribute(attributeName)) {
           const elemAttribute = elem.getAttribute(attributeName);
           if (!elemAttribute) return;
 
@@ -61,7 +62,7 @@ export const addBaseUrl = ({
             entityId,
             path,
           );
-          const apiOrigin = await techdocsStorageApi.getApiOrigin();
+
           if (isSvgNeedingInlining(attributeName, elemAttribute, apiOrigin)) {
             try {
               const svg = await fetch(newValue, { credentials: 'include' });
@@ -76,13 +77,16 @@ export const addBaseUrl = ({
           } else {
             elem.setAttribute(attributeName, newValue);
           }
-        });
+        }
+      }
     };
 
-    updateDom<HTMLImageElement>(dom.querySelectorAll('img'), 'src');
-    updateDom<HTMLScriptElement>(dom.querySelectorAll('script'), 'src');
-    updateDom<HTMLLinkElement>(dom.querySelectorAll('link'), 'href');
-    updateDom<HTMLAnchorElement>(dom.querySelectorAll('a[download]'), 'href');
+    await Promise.all([
+      updateDom<HTMLImageElement>(dom.querySelectorAll('img'), 'src'),
+      updateDom<HTMLScriptElement>(dom.querySelectorAll('script'), 'src'),
+      updateDom<HTMLLinkElement>(dom.querySelectorAll('link'), 'href'),
+      updateDom<HTMLAnchorElement>(dom.querySelectorAll('a[download]'), 'href'),
+    ]);
 
     return dom;
   };

plugins/techdocs/src/reader/transformers/addGitFeedbackLink.test.ts

@@ -28,8 +28,8 @@ const integrations = ScmIntegrations.fromConfig(
 );
 
 describe('addGitFeedbackLink', () => {
-  it('adds a feedback link when a Gitlab source edit link is available', () => {
-    const shadowDom = createTestShadowDom(
+  it('adds a feedback link when a Gitlab source edit link is available', async () => {
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>
@@ -53,8 +53,8 @@ describe('addGitFeedbackLink', () => {
     );
   });
 
-  it('adds a feedback link when a Github source edit link is available', () => {
-    const shadowDom = createTestShadowDom(
+  it('adds a feedback link when a Github source edit link is available', async () => {
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>
@@ -78,8 +78,8 @@ describe('addGitFeedbackLink', () => {
     );
   });
 
-  it('does not add a feedback link when no source edit link is available', () => {
-    const shadowDom = createTestShadowDom(
+  it('does not add a feedback link when no source edit link is available', async () => {
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>
@@ -97,8 +97,8 @@ describe('addGitFeedbackLink', () => {
     expect(shadowDom.querySelector('#git-feedback-link')).toBeFalsy();
   });
 
-  it('does not add a feedback link when a Gitlab or Github source edit link is not available', () => {
-    const shadowDom = createTestShadowDom(
+  it('does not add a feedback link when a Gitlab or Github source edit link is not available', async () => {
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>
@@ -117,8 +117,8 @@ describe('addGitFeedbackLink', () => {
     expect(shadowDom.querySelector('#git-feedback-link')).toBeFalsy();
   });
 
-  it('adds a feedback link when a Gitlab or Github source edit link is not available but hostname matches an integrations host', () => {
-    const shadowDom = createTestShadowDom(
+  it('adds a feedback link when a Gitlab or Github source edit link is not available but hostname matches an integrations host', async () => {
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>

plugins/techdocs/src/reader/transformers/addLinkClickListener.test.ts

@@ -18,9 +18,9 @@ import { createTestShadowDom } from '../../test-utils';
 import { addLinkClickListener } from './addLinkClickListener';
 
 describe('addLinkClickListener', () => {
-  it('calls onClick when a link has been clicked', () => {
+  it('calls onClick when a link has been clicked', async () => {
     const fn = jest.fn();
-    const shadowDom = createTestShadowDom(
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>
@@ -45,9 +45,9 @@ describe('addLinkClickListener', () => {
     expect(fn).toHaveBeenCalledTimes(1);
   });
 
-  it('does not call onClick when a link links to another baseUrl', () => {
+  it('does not call onClick when a link links to another baseUrl', async () => {
     const fn = jest.fn();
-    const shadowDom = createTestShadowDom(
+    const shadowDom = await createTestShadowDom(
       `
       <!DOCTYPE html>
       <html>

plugins/techdocs/src/reader/transformers/index.test.ts

@@ -17,14 +17,14 @@
 import { Transformer, transform } from './transformer';
 
 describe('transform', () => {
-  it('calls the transformers', () => {
+  it('calls the transformers', async () => {
     const fn = jest.fn();
     const mockTransformer = (): Transformer => (dom: Element) => {
       fn(dom);
       return dom;
     };
 
-    transform('<html></html>', [mockTransformer()]);
+    await transform('<html></html>', [mockTransformer()]);
 
     expect(fn).toHaveBeenCalledTimes(1);
     expect(fn).toHaveBeenCalledWith(expect.any(Element));

plugins/techdocs/src/reader/transformers/injectCss.test.ts

@@ -15,10 +15,10 @@
  */
 
 import { createTestShadowDom } from '../../test-utils';
-import { injectCss } from '../transformers';
+import { injectCss } from './injectCss';
 
 describe('injectCss', () => {
-  it('should inject style with passed css in head', () => {
+  it('should inject style with passed css in head', async () => {
     const html = `
         <html>
             <head></head>
@@ -27,7 +27,7 @@ describe('injectCss', () => {
     `;
     const injectedCss = '* {background-color: #fff}';
 
-    const shadowDom = createTestShadowDom(html, {
+    const shadowDom = await createTestShadowDom(html, {
       preTransformers: [injectCss({ css: injectedCss })],
       postTransformers: [],
     });

plugins/techdocs/src/reader/transformers/onCssReady.test.ts

@@ -15,16 +15,15 @@
  */
 
 import {
-  createTestShadowDom,
-  mockStylesheetEventListener,
-  executeStylesheetEventListeners,
   clearStylesheetEventListeners,
+  createTestShadowDom,
+  executeStylesheetEventListeners,
+  mockStylesheetEventListener,
 } from '../../test-utils';
-import { onCssReady } from '../transformers';
+import { onCssReady } from './onCssReady';
 
-const docStorageUrl: Promise<string> = Promise.resolve(
-  'https://techdocs-mock-sites.storage.googleapis.com',
-);
+const docStorageUrl: string =
+  'https://techdocs-mock-sites.storage.googleapis.com';
 
 const fixture = `
   <link rel="stylesheet" href="${docStorageUrl}/test.css" />
@@ -48,11 +47,11 @@ describe('onCssReady', () => {
     clearStylesheetEventListeners();
   });
 
-  it('does not call onLoading and onLoaded without the onCssReady transformer', () => {
+  it('does not call onLoading and onLoaded without the onCssReady transformer', async () => {
     const onLoading = jest.fn();
     const onLoaded = jest.fn();
 
-    createTestShadowDom(fixture, {
+    await createTestShadowDom(fixture, {
       preTransformers: [],
       postTransformers: [],
     });
@@ -62,11 +61,11 @@ describe('onCssReady', () => {
     expect(onLoaded).not.toHaveBeenCalled();
   });
 
-  it('calls the onLoading and onLoaded correctly', () => {
+  it('calls the onLoading and onLoaded correctly', async () => {
     const onLoading = jest.fn();
     const onLoaded = jest.fn();
 
-    createTestShadowDom(fixture, {
+    await createTestShadowDom(fixture, {
       preTransformers: [],
       postTransformers: [
         onCssReady({

plugins/techdocs/src/reader/transformers/onCssReady.ts

@@ -17,7 +17,7 @@
 import type { Transformer } from './transformer';
 
 type OnCssReadyOptions = {
-  docStorageUrl: Promise<string>;
+  docStorageUrl: string;
   onLoading: (dom: Element) => void;
   onLoaded: (dom: Element) => void;
 };
@@ -30,9 +30,7 @@ export const onCssReady = ({
   return dom => {
     const cssPages = Array.from(
       dom.querySelectorAll('head > link[rel="stylesheet"]'),
-    ).filter(async elem =>
-      elem.getAttribute('href')?.startsWith(await docStorageUrl),
-    );
+    ).filter(elem => elem.getAttribute('href')?.startsWith(docStorageUrl));
 
     let count = cssPages.length;
 

plugins/techdocs/src/reader/transformers/removeMkdocsHeader.test.ts

@@ -18,20 +18,26 @@ import { createTestShadowDom, FIXTURES } from '../../test-utils';
 import { removeMkdocsHeader } from '../transformers';
 
 describe('removeMkdocsHeader', () => {
-  it('does not remove mkdocs header', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [],
-      postTransformers: [],
-    });
+  it('does not remove mkdocs header', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [],
+        postTransformers: [],
+      },
+    );
 
     expect(shadowDom.querySelector('.md-header')).toBeTruthy();
   });
 
-  it('does remove mkdocs header', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [removeMkdocsHeader()],
-      postTransformers: [],
-    });
+  it('does remove mkdocs header', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [removeMkdocsHeader()],
+        postTransformers: [],
+      },
+    );
 
     expect(shadowDom.querySelector('.md-header')).toBeFalsy();
   });

plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts

@@ -19,8 +19,8 @@ import { rewriteDocLinks } from '../transformers';
 import { normalizeUrl } from './rewriteDocLinks';
 
 describe('rewriteDocLinks', () => {
-  it('should not do anything', () => {
-    const shadowDom = createTestShadowDom(`
+  it('should not do anything', async () => {
+    const shadowDom = await createTestShadowDom(`
         <a href="http://example.org/">Test</a>
         <a href="../example">Test</a>
         <a href="example-docs">Test</a>
@@ -35,8 +35,8 @@ describe('rewriteDocLinks', () => {
     ]);
   });
 
-  it('should transform a href with localhost as baseUrl', () => {
-    const shadowDom = createTestShadowDom(
+  it('should transform a href with localhost as baseUrl', async () => {
+    const shadowDom = await createTestShadowDom(
       `
         <a href="http://example.org/">Test</a>
         <a href="../example">Test</a>
@@ -57,9 +57,9 @@ describe('rewriteDocLinks', () => {
     ]);
   });
 
-  it('should rewrite non-parseable URLs as text', () => {
+  it('should rewrite non-parseable URLs as text', async () => {
     const expectedText = `www.my-internet.[top-level-domain]/pathname/[URLkey]`;
-    const shadowDom = createTestShadowDom(
+    const shadowDom = await createTestShadowDom(
       `<a href="http://${expectedText}">${expectedText}</a>`,
       {
         preTransformers: [rewriteDocLinks()],

plugins/techdocs/src/reader/transformers/sanitizeDOM/index.test.ts

@@ -16,7 +16,7 @@
 
 import { createTestShadowDom, FIXTURES } from '../../../test-utils';
 import { Transformer } from '../index';
-import { sanitizeDOM } from '../sanitizeDOM';
+import { sanitizeDOM } from './index';
 
 const injectMaliciousLink = (): Transformer => dom => {
   const link = document.createElement('a');
@@ -27,55 +27,64 @@ const injectMaliciousLink = (): Transformer => dom => {
 };
 
 describe('sanitizeDOM', () => {
-  it('contains a script tag', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE);
+  it('contains a script tag', async () => {
+    const shadowDom = await createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE);
 
     expect(shadowDom.querySelectorAll('script').length).toBeGreaterThan(0);
   });
 
-  it('does not contain a script tag', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [sanitizeDOM()],
-      postTransformers: [],
-    });
+  it('does not contain a script tag', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [sanitizeDOM()],
+        postTransformers: [],
+      },
+    );
 
     expect(shadowDom.querySelectorAll('script').length).toBe(0);
   });
 
-  it('contains link with a onClick attribute', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [injectMaliciousLink()],
-      postTransformers: [],
-    });
+  it('contains link with a onClick attribute', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [injectMaliciousLink()],
+        postTransformers: [],
+      },
+    );
 
     expect(
       shadowDom.querySelector('#test-malicious-link')?.hasAttribute('onclick'),
     ).toBeTruthy();
   });
 
-  it('does not contain link with a onClick attribute', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [sanitizeDOM()],
-      postTransformers: [],
-    });
+  it('does not contain link with a onClick attribute', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [sanitizeDOM()],
+        postTransformers: [],
+      },
+    );
 
     expect(
       shadowDom.querySelector('#test-malicious-link')?.hasAttribute('onclick'),
     ).toBeFalsy();
   });
 
-  it('removes style tags', () => {
+  it('removes style tags', async () => {
     const html = `
       <html>
         <head>
-          <style>* {color: #f0f;}<style>
+          <style>* {color: #f0f;}</style>
         </head>
         <body>
         </body>
       </html>
     `;
 
-    const shadowDom = createTestShadowDom(html, {
+    const shadowDom = await createTestShadowDom(html, {
       preTransformers: [sanitizeDOM()],
       postTransformers: [],
     });
@@ -83,7 +92,7 @@ describe('sanitizeDOM', () => {
     expect(shadowDom.querySelectorAll('style').length).toEqual(0);
   });
 
-  it('does not remove link tags', () => {
+  it('does not remove link tags', async () => {
     const html = `
       <html>
         <head>
@@ -94,7 +103,7 @@ describe('sanitizeDOM', () => {
       </html>
     `;
 
-    const shadowDom = createTestShadowDom(html, {
+    const shadowDom = await createTestShadowDom(html, {
       preTransformers: [sanitizeDOM()],
       postTransformers: [],
     });

plugins/techdocs/src/reader/transformers/simplifyMkdocsFooter.test.ts

@@ -18,20 +18,26 @@ import { createTestShadowDom, FIXTURES } from '../../test-utils';
 import { simplifyMkdocsFooter } from './simplifyMkdocsFooter';
 
 describe('simplifyMkdocsFooter', () => {
-  it('does not remove mkdocs copyright', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [],
-      postTransformers: [],
-    });
+  it('does not remove mkdocs copyright', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [],
+        postTransformers: [],
+      },
+    );
 
     expect(shadowDom.querySelector('.md-footer-copyright')).toBeTruthy();
   });
 
-  it('does remove mkdocs copyright', () => {
-    const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, {
-      preTransformers: [simplifyMkdocsFooter()],
-      postTransformers: [],
-    });
+  it('does remove mkdocs copyright', async () => {
+    const shadowDom = await createTestShadowDom(
+      FIXTURES.FIXTURE_STANDARD_PAGE,
+      {
+        preTransformers: [simplifyMkdocsFooter()],
+        postTransformers: [],
+      },
+    );
 
     expect(shadowDom.querySelector('.md-footer-copyright')).toBeFalsy();
   });

plugins/techdocs/src/reader/transformers/transformer.ts

@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-export type Transformer = (dom: Element) => Element;
+export type Transformer = (dom: Element) => Element | Promise<Element>;
 
-export const transform = (
+export const transform = async (
   html: string | Element,
   transformers: Transformer[],
-): Element => {
+): Promise<Element> => {
   let dom: Element;
 
   if (typeof html === 'string') {
@@ -30,9 +30,9 @@ export const transform = (
     throw new Error('dom is not a recognized type');
   }
 
-  transformers.forEach(transformer => {
-    dom = transformer(dom);
-  });
+  for (const transformer of transformers) {
+    dom = await transformer(dom);
+  }
 
   return dom;
 };

plugins/techdocs/src/test-utils/shadowDom.ts

@@ -22,13 +22,13 @@ export type CreateTestShadowDomOptions = {
   postTransformers: Transformer[];
 };
 
-export const createTestShadowDom = (
+export const createTestShadowDom = async (
   fixture: string,
   opts: CreateTestShadowDomOptions = {
     preTransformers: [],
     postTransformers: [],
   },
-): ShadowRoot => {
+): Promise<ShadowRoot> => {
   const divElement = document.createElement('div');
   divElement.attachShadow({ mode: 'open' });
   document.body.appendChild(divElement);
@@ -39,7 +39,7 @@ export const createTestShadowDom = (
     'text/html',
   ).documentElement;
   if (opts.preTransformers) {
-    dom = transformer(dom, opts.preTransformers);
+    dom = await transformer(dom, opts.preTransformers);
   }
 
   // Mount the UI
@@ -47,7 +47,7 @@ export const createTestShadowDom = (
 
   // Transformers after the UI is rendered
   if (opts.postTransformers) {
-    transformer(dom, opts.postTransformers);
+    await transformer(dom, opts.postTransformers);
   }
 
   return divElement.shadowRoot!;