Move packages/techdocs-common -> plugins/techdocs-node

Signed-off-by: Eric Peterson <ericpeterson@spotify.com>

plugins/techdocs-node/src/helpers.test.ts

@@ -0,0 +1,326 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ReadTreeResponse,
+  SearchResponse,
+  UrlReader,
+} from '@backstage/backend-common';
+import { Entity, getEntitySourceLocation } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import os from 'os';
+import path from 'path';
+import { Readable } from 'stream';
+import {
+  getDocFilesFromRepository,
+  getLocationForEntity,
+  parseReferenceAnnotation,
+  transformDirLocation,
+} from './helpers';
+
+jest.mock('@backstage/catalog-model', () => ({
+  ...jest.requireActual('@backstage/catalog-model'),
+  getEntitySourceLocation: jest.fn(),
+}));
+
+const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+const entityBase: Entity = {
+  metadata: {
+    namespace: 'default',
+    name: 'mytestcomponent',
+    description: 'A component for testing',
+  },
+  apiVersion: 'backstage.io/v1alpha1',
+  kind: 'Component',
+  spec: {
+    type: 'documentation',
+    lifecycle: 'experimental',
+    owner: 'testuser',
+  },
+};
+
+const metadataBase = {
+  namespace: 'default',
+  name: 'mytestcomponent',
+  description: 'A component for testing',
+};
+
+const goodAnnotation = {
+  annotations: {
+    'backstage.io/techdocs-ref':
+      'url:https://github.com/backstage/backstage/blob/master/subfolder/',
+  },
+};
+
+const mockEntityWithAnnotation: Entity = {
+  ...entityBase,
+  ...{
+    metadata: {
+      ...metadataBase,
+      ...goodAnnotation,
+    },
+  },
+};
+
+const badAnnotation = {
+  annotations: {
+    'backstage.io/techdocs-ref': 'bad-annotation',
+  },
+};
+
+const mockEntityWithBadAnnotation: Entity = {
+  ...entityBase,
+  ...{
+    metadata: {
+      ...metadataBase,
+      ...badAnnotation,
+    },
+  },
+};
+
+const scmIntegrations = ScmIntegrations.fromConfig(new ConfigReader({}));
+
+afterEach(() => jest.resetAllMocks());
+
+describe('parseReferenceAnnotation', () => {
+  it('should parse annotation', () => {
+    const parsedLocationAnnotation = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      mockEntityWithAnnotation,
+    );
+    expect(parsedLocationAnnotation.type).toBe('url');
+    expect(parsedLocationAnnotation.target).toBe(
+      'https://github.com/backstage/backstage/blob/master/subfolder/',
+    );
+  });
+
+  it('should throw error without annotation', () => {
+    expect(() => {
+      parseReferenceAnnotation('backstage.io/techdocs-ref', entityBase);
+    }).toThrow(/No location annotation/);
+  });
+
+  it('should throw error with bad annotation', () => {
+    expect(() => {
+      parseReferenceAnnotation(
+        'backstage.io/techdocs-ref',
+        mockEntityWithBadAnnotation,
+      );
+    }).toThrow(/Unable to parse/);
+  });
+});
+
+describe('transformDirLocation', () => {
+  it.each`
+    techdocsRef           | target
+    ${'dir:.'}            | ${'https://my-url/folder/'}
+    ${'dir:./sub-folder'} | ${'https://my-url/folder/sub-folder'}
+  `(
+    'should transform "$techdocsRef" for url type locations',
+    ({ techdocsRef, target }) => {
+      (getEntitySourceLocation as jest.Mock).mockReturnValue({
+        type: 'url',
+        target: 'https://my-url/folder/',
+      });
+
+      const entity = {
+        apiVersion: '1',
+        kind: 'Component',
+        metadata: {
+          name: 'test',
+          annotations: {
+            'backstage.io/techdocs-ref': techdocsRef,
+          },
+        },
+      };
+
+      const result = transformDirLocation(
+        entity,
+        parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
+        scmIntegrations,
+      );
+
+      expect(result).toEqual({ type: 'url', target });
+    },
+  );
+
+  it.each`
+    techdocsRef           | target
+    ${'dir:.'}            | ${path.join(rootDir, 'working-copy')}
+    ${'dir:./sub-folder'} | ${path.join(rootDir, 'working-copy', 'sub-folder')}
+  `(
+    'should transform "$techdocsRef" for file type locations',
+    ({ techdocsRef, target }) => {
+      (getEntitySourceLocation as jest.Mock).mockReturnValue({
+        type: 'file',
+        target: path.join(rootDir, 'working-copy', 'catalog-info.yaml'),
+      });
+
+      const entity = {
+        apiVersion: '1',
+        kind: 'Component',
+        metadata: {
+          name: 'test',
+          annotations: {
+            'backstage.io/techdocs-ref': techdocsRef,
+          },
+        },
+      };
+
+      const result = transformDirLocation(
+        entity,
+        parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
+        scmIntegrations,
+      );
+
+      expect(result).toEqual({ type: 'dir', target });
+    },
+  );
+
+  it('should reject unsafe file location', () => {
+    (getEntitySourceLocation as jest.Mock).mockReturnValue({
+      type: 'file',
+      target: '/tmp/catalog-info.yaml',
+    });
+
+    const entity = {
+      apiVersion: '1',
+      kind: 'Component',
+      metadata: {
+        name: 'test',
+        annotations: {
+          'backstage.io/techdocs-ref': 'dir:..',
+        },
+      },
+    };
+
+    expect(() =>
+      transformDirLocation(
+        entity,
+        parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
+        scmIntegrations,
+      ),
+    ).toThrow(
+      /Relative path is not allowed to refer to a directory outside its parent/,
+    );
+  });
+
+  it('should reject other location types', () => {
+    (getEntitySourceLocation as jest.Mock).mockReturnValue({
+      type: 'other',
+      target: '/',
+    });
+
+    const entity = {
+      apiVersion: '1',
+      kind: 'Component',
+      metadata: {
+        name: 'test',
+        annotations: {
+          'backstage.io/techdocs-ref': 'dir:.',
+        },
+      },
+    };
+
+    expect(() =>
+      transformDirLocation(
+        entity,
+        parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
+        scmIntegrations,
+      ),
+    ).toThrow(/Unable to resolve location type other/);
+  });
+});
+
+describe('getLocationForEntity', () => {
+  it('should handle dir locations', () => {
+    (getEntitySourceLocation as jest.Mock).mockReturnValue({
+      type: 'url',
+      target: 'https://my-url/folder/',
+    });
+
+    const entity = {
+      apiVersion: '1',
+      kind: 'Component',
+      metadata: {
+        name: 'test',
+        annotations: {
+          'backstage.io/techdocs-ref': 'dir:.',
+        },
+      },
+    };
+
+    const parsedLocationAnnotation = getLocationForEntity(
+      entity,
+      scmIntegrations,
+    );
+    expect(parsedLocationAnnotation.type).toBe('url');
+    expect(parsedLocationAnnotation.target).toBe('https://my-url/folder/');
+  });
+
+  it('should get location for entity', () => {
+    const parsedLocationAnnotation = getLocationForEntity(
+      mockEntityWithAnnotation,
+      scmIntegrations,
+    );
+    expect(parsedLocationAnnotation.type).toBe('url');
+    expect(parsedLocationAnnotation.target).toBe(
+      'https://github.com/backstage/backstage/blob/master/subfolder/',
+    );
+  });
+});
+
+describe('getDocFilesFromRepository', () => {
+  it('should read a remote directory using UrlReader.readTree', async () => {
+    class MockUrlReader implements UrlReader {
+      async read() {
+        return Buffer.from('mock');
+      }
+
+      async readTree(): Promise<ReadTreeResponse> {
+        return {
+          dir: async () => {
+            return '/tmp/testfolder';
+          },
+          files: async () => {
+            return [];
+          },
+          archive: async () => {
+            return Readable.from('');
+          },
+          etag: 'etag123abc',
+        };
+      }
+
+      async search(): Promise<SearchResponse> {
+        return {
+          etag: '',
+          files: [],
+        };
+      }
+    }
+
+    const output = await getDocFilesFromRepository(
+      new MockUrlReader(),
+      mockEntityWithAnnotation,
+    );
+
+    expect(output.preparedDir).toBe('/tmp/testfolder');
+    expect(output.etag).toBe('etag123abc');
+  });
+});

plugins/techdocs-node/src/helpers.ts

@@ -0,0 +1,167 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common';
+import {
+  Entity,
+  getEntitySourceLocation,
+  parseLocationRef,
+} from '@backstage/catalog-model';
+import { InputError } from '@backstage/errors';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import path from 'path';
+import { Logger } from 'winston';
+import { PreparerResponse, RemoteProtocol } from './stages/prepare/types';
+
+/**
+ * Parsed location annotation
+ * @public
+ */
+export type ParsedLocationAnnotation = {
+  type: RemoteProtocol;
+  target: string;
+};
+
+/**
+ * Returns a parset locations annotation
+ * @public
+ * @param annotationName - The name of the annotation in the entity metadata
+ * @param entity - A TechDocs entity instance
+ */
+export const parseReferenceAnnotation = (
+  annotationName: string,
+  entity: Entity,
+): ParsedLocationAnnotation => {
+  const annotation = entity.metadata.annotations?.[annotationName];
+  if (!annotation) {
+    throw new InputError(
+      `No location annotation provided in entity: ${entity.metadata.name}`,
+    );
+  }
+
+  const { type, target } = parseLocationRef(annotation);
+  return {
+    type: type as RemoteProtocol,
+    target,
+  };
+};
+
+/**
+ * TechDocs references of type `dir` are relative the source location of the entity.
+ * This function transforms relative references to absolute ones, based on the
+ * location the entity was ingested from. If the entity was registered by a `url`
+ * location, it returns a `url` location with a resolved target that points to the
+ * targeted subfolder. If the entity was registered by a `file` location, it returns
+ * an absolute `dir` location.
+ * @public
+ * @param entity - the entity with annotations
+ * @param dirAnnotation - the parsed techdocs-ref annotation of type 'dir'
+ * @param scmIntegrations - access to the scmIntegration to do url transformations
+ * @throws if the entity doesn't specify a `dir` location or is ingested from an unsupported location.
+ * @returns the transformed location with an absolute target.
+ */
+export const transformDirLocation = (
+  entity: Entity,
+  dirAnnotation: ParsedLocationAnnotation,
+  scmIntegrations: ScmIntegrationRegistry,
+): { type: 'dir' | 'url'; target: string } => {
+  const location = getEntitySourceLocation(entity);
+
+  switch (location.type) {
+    case 'url': {
+      const target = scmIntegrations.resolveUrl({
+        url: dirAnnotation.target,
+        base: location.target,
+      });
+
+      return {
+        type: 'url',
+        target,
+      };
+    }
+
+    case 'file': {
+      // only permit targets in the same folder as the target of the `file` location!
+      const target = resolveSafeChildPath(
+        path.dirname(location.target),
+        dirAnnotation.target,
+      );
+
+      return {
+        type: 'dir',
+        target,
+      };
+    }
+
+    default:
+      throw new InputError(`Unable to resolve location type ${location.type}`);
+  }
+};
+
+/**
+ * Returns a entity reference based on the TechDocs annotation type
+ * @public
+ * @param entity - A TechDocs instance
+ * @param scmIntegration - An implementation for  SCM integration API
+ */
+export const getLocationForEntity = (
+  entity: Entity,
+  scmIntegration: ScmIntegrationRegistry,
+): ParsedLocationAnnotation => {
+  const annotation = parseReferenceAnnotation(
+    'backstage.io/techdocs-ref',
+    entity,
+  );
+
+  switch (annotation.type) {
+    case 'url':
+      return annotation;
+    case 'dir':
+      return transformDirLocation(entity, annotation, scmIntegration);
+    default:
+      throw new Error(`Invalid reference annotation ${annotation.type}`);
+  }
+};
+
+/**
+ * Returns a preparer response {@link PreparerResponse}
+ * @public
+ * @param reader - Read a tree of files from a repository
+ * @param entity - A TechDocs entity instance
+ * @param opts - Options for configuring the reader, e.g. logger, etag, etc.
+ */
+export const getDocFilesFromRepository = async (
+  reader: UrlReader,
+  entity: Entity,
+  opts?: { etag?: string; logger?: Logger },
+): Promise<PreparerResponse> => {
+  const { target } = parseReferenceAnnotation(
+    'backstage.io/techdocs-ref',
+    entity,
+  );
+
+  opts?.logger?.debug(`Reading files from ${target}`);
+  // readTree will throw NotModifiedError if etag has not changed.
+  const readTreeResponse = await reader.readTree(target, { etag: opts?.etag });
+  const preparedDir = await readTreeResponse.dir();
+
+  opts?.logger?.debug(`Tree downloaded and stored at ${preparedDir}`);
+
+  return {
+    preparedDir,
+    etag: readTreeResponse.etag,
+  };
+};

plugins/techdocs-node/src/index.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli
+ *
+ * @packageDocumentation
+ */
+
+export * from './stages';
+export * from './helpers';
+export * from './techdocsTypes';

plugins/techdocs-node/src/setupTests.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { StorageFilesMock } from './testUtils/StorageFilesMock';
+
+(global as any).rootDir = StorageFilesMock.rootDir;
+(global as any).storageFilesMock = new StorageFilesMock();

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs.yml

@@ -0,0 +1,2 @@
+site_name: Test site name
+site_description: Test site description

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_invalid_doc_dir.yml

@@ -0,0 +1,3 @@
+site_name: Test site name
+site_description: Test site description
+docs_dir: /etc

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_invalid_doc_dir2.yml

@@ -0,0 +1,3 @@
+site_name: Test site name
+site_description: Test site description
+docs_dir: ../../etc/

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_valid_doc_dir.yml

@@ -0,0 +1,3 @@
+site_name: Test site name
+site_description: Test site description
+docs_dir: docs/

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml

@@ -0,0 +1,6 @@
+site_name: Test site name
+site_description: Test site description
+docs_dir: docs/
+plugins:
+  - not-techdocs-core
+  - also-not-techdocs-core

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_comments.yml

@@ -0,0 +1,3 @@
+site_name: Test site name
+site_description: Test site description
+# This is a comment that is removed after editing

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_edit_uri.yml

@@ -0,0 +1,4 @@
+site_name: Test site name
+site_description: Test site description
+
+edit_uri: https://github.com/backstage/backstage/edit/main/docs

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_extensions.yml

@@ -0,0 +1,7 @@
+site_name: Test site name
+site_description: Test site description
+
+markdown_extensions:
+  - pymdownx.emoji:
+      emoji_index: !!python/name:materialx.emoji.twemoji
+      emoji_generator: !!python/name:materialx.emoji.to_svg

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_repo_url.yml

@@ -0,0 +1,4 @@
+site_name: Test site name
+site_description: Test site description
+
+repo_url: https://github.com/backstage/backstage

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml

@@ -0,0 +1,5 @@
+site_name: Test site name
+site_description: Test site description
+# This is a comment that is removed after editing
+plugins:
+  - techdocs-core

plugins/techdocs-node/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml

@@ -0,0 +1,3 @@
+site_name: Test site name
+site_description: Test site description
+docs_dir: docs/

plugins/techdocs-node/src/stages/generate/generators.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ContainerRunner, getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { Generators } from './generators';
+import { TechdocsGenerator } from './techdocs';
+
+const logger = getVoidLogger();
+
+const mockEntity = {
+  apiVersion: 'version',
+  kind: 'TestKind',
+  metadata: {
+    name: 'testName',
+  },
+};
+
+describe('generators', () => {
+  const containerRunner: jest.Mocked<ContainerRunner> = {
+    runContainer: jest.fn(),
+  };
+
+  it('should return error if no generator is registered', async () => {
+    const generators = new Generators();
+
+    expect(() => generators.get(mockEntity)).toThrowError(
+      'No generator registered for entity: "techdocs"',
+    );
+  });
+
+  it('should return correct registered generator', async () => {
+    const generators = new Generators();
+    const techdocs = TechdocsGenerator.fromConfig(new ConfigReader({}), {
+      logger,
+      containerRunner,
+    });
+
+    generators.register('techdocs', techdocs);
+
+    expect(generators.get(mockEntity)).toBe(techdocs);
+  });
+});

plugins/techdocs-node/src/stages/generate/generators.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ContainerRunner } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import { getGeneratorKey } from './helpers';
+import { TechdocsGenerator } from './techdocs';
+import {
+  GeneratorBase,
+  GeneratorBuilder,
+  SupportedGeneratorKey,
+} from './types';
+
+/**
+ * Collection of docs generators
+ * @public
+ */
+export class Generators implements GeneratorBuilder {
+  private generatorMap = new Map<SupportedGeneratorKey, GeneratorBase>();
+
+  /**
+   * Returns a generators instance containing a generator for TechDocs
+   * @param config - A Backstage configuration
+   * @param options - Options to configure the TechDocs generator
+   */
+  static async fromConfig(
+    config: Config,
+    options: { logger: Logger; containerRunner: ContainerRunner },
+  ): Promise<GeneratorBuilder> {
+    const generators = new Generators();
+
+    const techdocsGenerator = TechdocsGenerator.fromConfig(config, options);
+    generators.register('techdocs', techdocsGenerator);
+
+    return generators;
+  }
+
+  /**
+   * Register a generator in the generators collection
+   * @param generatorKey - Unique identifier for the generator
+   * @param generator - The generator instance to register
+   */
+  register(generatorKey: SupportedGeneratorKey, generator: GeneratorBase) {
+    this.generatorMap.set(generatorKey, generator);
+  }
+
+  /**
+   * Returns the generator for a given TechDocs entity
+   * @param entity - A TechDocs entity instance
+   */
+  get(entity: Entity): GeneratorBase {
+    const generatorKey = getGeneratorKey(entity);
+    const generator = this.generatorMap.get(generatorKey);
+
+    if (!generator) {
+      throw new Error(`No generator registered for entity: "${generatorKey}"`);
+    }
+
+    return generator;
+  }
+}

plugins/techdocs-node/src/stages/generate/helpers.test.ts

@@ -0,0 +1,591 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import os from 'os';
+import path, { resolve as resolvePath } from 'path';
+import { ParsedLocationAnnotation } from '../../helpers';
+import {
+  createOrUpdateMetadata,
+  getGeneratorKey,
+  getMkdocsYml,
+  getRepoUrlFromLocationAnnotation,
+  patchIndexPreBuild,
+  storeEtagMetadata,
+  validateMkdocsYaml,
+} from './helpers';
+import {
+  patchMkdocsYmlPreBuild,
+  pathMkdocsYmlWithTechdocsPlugin,
+} from './mkDocsPatchers';
+import yaml from 'js-yaml';
+
+const mockEntity = {
+  apiVersion: 'version',
+  kind: 'TestKind',
+  metadata: {
+    name: 'testName',
+  },
+};
+
+const mkdocsYml = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs.yml'),
+);
+const mkdocsYmlWithExtensions = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_extensions.yml'),
+);
+const mkdocsYmlWithRepoUrl = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'),
+);
+const mkdocsYmlWithEditUri = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_edit_uri.yml'),
+);
+const mkdocsYmlWithValidDocDir = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_valid_doc_dir.yml'),
+);
+const mkdocsYmlWithInvalidDocDir = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_invalid_doc_dir.yml'),
+);
+const mkdocsYmlWithInvalidDocDir2 = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_invalid_doc_dir2.yml'),
+);
+const mkdocsYmlWithComments = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_comments.yml'),
+);
+const mkdocsYmlWithTechdocsPlugins = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_techdocs_plugin.yml'),
+);
+const mkdocsYmlWithoutPlugins = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_without_plugins.yml'),
+);
+const mkdocsYmlWithAdditionalPlugins = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_additional_plugins.yml'),
+);
+const mockLogger = getVoidLogger();
+const warn = jest.spyOn(mockLogger, 'warn');
+
+const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+const scmIntegrations = ScmIntegrations.fromConfig(new ConfigReader({}));
+
+describe('helpers', () => {
+  describe('getGeneratorKey', () => {
+    it('should return techdocs as the only generator key', () => {
+      const key = getGeneratorKey(mockEntity);
+      expect(key).toBe('techdocs');
+    });
+  });
+
+  describe('getRepoUrlFromLocationAnnotation', () => {
+    it.each`
+      url                                                                        | repo_url                                    | edit_uri
+      ${'https://github.com/backstage/backstage'}                                | ${'https://github.com/backstage/backstage'} | ${undefined}
+      ${'https://github.com/backstage/backstage/tree/main/examples/techdocs/'}   | ${undefined}                                | ${'https://github.com/backstage/backstage/edit/main/examples/techdocs/docs'}
+      ${'https://github.com/backstage/backstage/tree/main/'}                     | ${undefined}                                | ${'https://github.com/backstage/backstage/edit/main/docs'}
+      ${'https://gitlab.com/backstage/backstage'}                                | ${'https://gitlab.com/backstage/backstage'} | ${undefined}
+      ${'https://gitlab.com/backstage/backstage/-/blob/main/examples/techdocs/'} | ${undefined}                                | ${'https://gitlab.com/backstage/backstage/-/edit/main/examples/techdocs/docs'}
+      ${'https://gitlab.com/backstage/backstage/-/blob/main/'}                   | ${undefined}                                | ${'https://gitlab.com/backstage/backstage/-/edit/main/docs'}
+    `('should convert $url', ({ url: target, repo_url, edit_uri }) => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target,
+      };
+
+      expect(
+        getRepoUrlFromLocationAnnotation(
+          parsedLocationAnnotation,
+          scmIntegrations,
+        ),
+      ).toEqual({ repo_url, edit_uri });
+    });
+
+    it.each`
+      url                                                                        | edit_uri
+      ${'https://github.com/backstage/backstage/tree/main/examples/techdocs/'}   | ${'https://github.com/backstage/backstage/edit/main/examples/techdocs/custom/folder'}
+      ${'https://github.com/backstage/backstage/tree/main/'}                     | ${'https://github.com/backstage/backstage/edit/main/custom/folder'}
+      ${'https://gitlab.com/backstage/backstage/-/blob/main/examples/techdocs/'} | ${'https://gitlab.com/backstage/backstage/-/edit/main/examples/techdocs/custom/folder'}
+      ${'https://gitlab.com/backstage/backstage/-/blob/main/'}                   | ${'https://gitlab.com/backstage/backstage/-/edit/main/custom/folder'}
+    `(
+      'should convert $url with custom docsFolder',
+      ({ url: target, edit_uri }) => {
+        const parsedLocationAnnotation: ParsedLocationAnnotation = {
+          type: 'url',
+          target,
+        };
+
+        expect(
+          getRepoUrlFromLocationAnnotation(
+            parsedLocationAnnotation,
+            scmIntegrations,
+            './custom/folder',
+          ),
+        ).toEqual({ edit_uri });
+      },
+    );
+
+    it.each`
+      url
+      ${'https://bitbucket.org/backstage/backstage/src/master/examples/techdocs/'}
+      ${'https://bitbucket.org/backstage/backstage/src/master/'}
+      ${'https://dev.azure.com/organization/project/_git/repository?path=%2Fexamples%2Ftechdocs'}
+      ${'https://dev.azure.com/organization/project/_git/repository?path=%2F'}
+    `('should ignore $url', ({ url: target }) => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target,
+      };
+
+      expect(
+        getRepoUrlFromLocationAnnotation(
+          parsedLocationAnnotation,
+          scmIntegrations,
+        ),
+      ).toEqual({});
+    });
+
+    it('should ignore unsupported location type', () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'dir',
+        target: '/home/user/workspace/docs-repository',
+      };
+
+      expect(
+        getRepoUrlFromLocationAnnotation(
+          parsedLocationAnnotation,
+          scmIntegrations,
+        ),
+      ).toEqual({});
+    });
+  });
+
+  describe('patchMkdocsYmlPreBuild', () => {
+    beforeEach(() => {
+      mockFs({
+        '/mkdocs.yml': mkdocsYml,
+        '/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
+        '/mkdocs_with_edit_uri.yml': mkdocsYmlWithEditUri,
+        '/mkdocs_with_extensions.yml': mkdocsYmlWithExtensions,
+        '/mkdocs_with_comments.yml': mkdocsYmlWithComments,
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should add edit_uri to mkdocs.yml', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target: 'https://github.com/backstage/backstage',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+        scmIntegrations,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        'repo_url: https://github.com/backstage/backstage',
+      );
+    });
+
+    it('should add repo_url to mkdocs.yml that contains custom yaml tags', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target: 'https://github.com/backstage/backstage',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs_with_extensions.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+        scmIntegrations,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_with_extensions.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        'repo_url: https://github.com/backstage/backstage',
+      );
+      expect(updatedMkdocsYml.toString()).toContain(
+        "emoji_index: !!python/name:materialx.emoji.twemoji ''",
+      );
+    });
+
+    it('should not override existing repo_url in mkdocs.yml', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target: 'https://github.com/neworg/newrepo',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs_with_repo_url.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+        scmIntegrations,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_with_repo_url.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        'repo_url: https://github.com/backstage/backstage',
+      );
+      expect(updatedMkdocsYml.toString()).not.toContain(
+        'repo_url: https://github.com/neworg/newrepo',
+      );
+    });
+
+    it('should not override existing edit_uri in mkdocs.yml', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'url',
+        target: 'https://github.com/neworg/newrepo',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs_with_edit_uri.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+        scmIntegrations,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_with_edit_uri.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        'edit_uri: https://github.com/backstage/backstage/edit/main/docs',
+      );
+      expect(updatedMkdocsYml.toString()).not.toContain(
+        'https://github.com/neworg/newrepo',
+      );
+    });
+
+    it('should not update mkdocs.yml if nothing should be changed', async () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'dir',
+        target: '/unsupported/path',
+      };
+
+      await patchMkdocsYmlPreBuild(
+        '/mkdocs_with_comments.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+        scmIntegrations,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_with_comments.yml');
+
+      expect(updatedMkdocsYml.toString()).toContain(
+        '# This is a comment that is removed after editing',
+      );
+      expect(updatedMkdocsYml.toString()).not.toContain('edit_uri');
+      expect(updatedMkdocsYml.toString()).not.toContain('repo_url');
+    });
+  });
+
+  describe('pathMkdocsYmlWithTechdocsPlugin', () => {
+    beforeEach(() => {
+      mockFs({
+        '/mkdocs_with_techdocs_plugin.yml': mkdocsYmlWithTechdocsPlugins,
+        '/mkdocs_without_plugins.yml': mkdocsYmlWithoutPlugins,
+        '/mkdocs_with_additional_plugins.yml': mkdocsYmlWithAdditionalPlugins,
+      });
+    });
+    it('should not add additional plugins if techdocs exists already in mkdocs file', async () => {
+      await pathMkdocsYmlWithTechdocsPlugin(
+        '/mkdocs_with_techdocs_plugin.yml',
+        mockLogger,
+      );
+
+      const updatedMkdocsYml = await fs.readFile(
+        '/mkdocs_with_techdocs_plugin.yml',
+      );
+      const parsedYml = yaml.load(updatedMkdocsYml.toString()) as {
+        plugins: string[];
+      };
+      expect(parsedYml.plugins).toHaveLength(1);
+      expect(parsedYml.plugins).toContain('techdocs-core');
+    });
+    it("should add the needed plugin if it doesn't exist in mkdocs file", async () => {
+      await pathMkdocsYmlWithTechdocsPlugin(
+        '/mkdocs_without_plugins.yml',
+        mockLogger,
+      );
+
+      const updatedMkdocsYml = await fs.readFile('/mkdocs_without_plugins.yml');
+      const parsedYml = yaml.load(updatedMkdocsYml.toString()) as {
+        plugins: string[];
+      };
+      expect(parsedYml.plugins).toHaveLength(1);
+      expect(parsedYml.plugins).toContain('techdocs-core');
+    });
+    it('should not override existing plugins', async () => {
+      await pathMkdocsYmlWithTechdocsPlugin(
+        '/mkdocs_with_additional_plugins.yml',
+        mockLogger,
+      );
+      const updatedMkdocsYml = await fs.readFile(
+        '/mkdocs_with_additional_plugins.yml',
+      );
+      const parsedYml = yaml.load(updatedMkdocsYml.toString()) as {
+        plugins: string[];
+      };
+      expect(parsedYml.plugins).toHaveLength(3);
+      expect(parsedYml.plugins).toContain('techdocs-core');
+      expect(parsedYml.plugins).toContain('not-techdocs-core');
+      expect(parsedYml.plugins).toContain('also-not-techdocs-core');
+    });
+  });
+
+  describe('patchIndexPreBuild', () => {
+    afterEach(() => {
+      warn.mockClear();
+    });
+    it('should have no effect if docs/index.md exists', async () => {
+      mockFs({
+        '/docs/index.md': 'index.md content',
+        '/docs/README.md': 'docs/README.md content',
+      });
+
+      await patchIndexPreBuild({ inputDir: '/', logger: mockLogger });
+
+      expect(fs.readFileSync('/docs/index.md', 'utf-8')).toEqual(
+        'index.md content',
+      );
+      expect(warn).not.toHaveBeenCalledWith();
+      mockFs.restore();
+    });
+
+    it("should use docs/README.md if docs/index.md doesn't exists", async () => {
+      mockFs({
+        '/docs/README.md': 'docs/README.md content',
+        '/README.md': 'main README.md content',
+      });
+
+      await patchIndexPreBuild({ inputDir: '/', logger: mockLogger });
+
+      expect(fs.readFileSync('/docs/index.md', 'utf-8')).toEqual(
+        'docs/README.md content',
+      );
+      expect(warn.mock.calls).toEqual([
+        [`${path.normalize('docs/index.md')} not found.`],
+      ]);
+      mockFs.restore();
+    });
+
+    it('should use README.md if neither docs/index.md or docs/README.md exist', async () => {
+      mockFs({
+        '/README.md': 'main README.md content',
+      });
+
+      await patchIndexPreBuild({ inputDir: '/', logger: mockLogger });
+
+      expect(fs.readFileSync('/docs/index.md', 'utf-8')).toEqual(
+        'main README.md content',
+      );
+      expect(warn.mock.calls).toEqual([
+        [`${path.normalize('docs/index.md')} not found.`],
+        [`${path.normalize('docs/README.md')} not found.`],
+        [`${path.normalize('docs/readme.md')} not found.`],
+      ]);
+      mockFs.restore();
+    });
+
+    it('should not use any file as index.md if no one matches the requirements', async () => {
+      mockFs({});
+
+      await patchIndexPreBuild({ inputDir: '/', logger: mockLogger });
+
+      expect(() => fs.readFileSync('/docs/index.md', 'utf-8')).toThrow();
+      const paths = [
+        path.normalize('docs/index.md'),
+        path.normalize('docs/README.md'),
+        path.normalize('docs/readme.md'),
+        'README.md',
+        'readme.md',
+      ];
+      expect(warn.mock.calls).toEqual([
+        ...paths.map(p => [`${p} not found.`]),
+        [
+          `Could not find any techdocs' index file. Please make sure at least one of ${paths
+            .map(p => path.sep + p)
+            .join(' ')} exists.`,
+        ],
+      ]);
+      mockFs.restore();
+    });
+  });
+
+  describe('addBuildTimestampMetadata', () => {
+    const mockFiles = {
+      'invalid_techdocs_metadata.json': 'dsds',
+      'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
+    };
+
+    beforeEach(() => {
+      mockFs.restore();
+      mockFs({
+        [rootDir]: mockFiles,
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should create the file if it does not exist', async () => {
+      const filePath = path.join(rootDir, 'wrong_techdocs_metadata.json');
+      await createOrUpdateMetadata(filePath, mockLogger);
+
+      // Check if the file exists
+      await expect(
+        fs.access(filePath, fs.constants.F_OK),
+      ).resolves.not.toThrowError();
+    });
+
+    it('should throw error when the JSON is invalid', async () => {
+      const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json');
+
+      await expect(
+        createOrUpdateMetadata(filePath, mockLogger),
+      ).rejects.toThrowError('Unexpected token d in JSON at position 0');
+    });
+
+    it('should add build timestamp to the metadata json', async () => {
+      const filePath = path.join(rootDir, 'techdocs_metadata.json');
+
+      await createOrUpdateMetadata(filePath, mockLogger);
+
+      const json = await fs.readJson(filePath);
+      expect(json.build_timestamp).toBeLessThanOrEqual(Date.now());
+    });
+
+    it('should add list of files to the metadata json', async () => {
+      const filePath = path.join(rootDir, 'techdocs_metadata.json');
+
+      await createOrUpdateMetadata(filePath, mockLogger);
+
+      const json = await fs.readJson(filePath);
+      expect(json.files[0]).toEqual(Object.keys(mockFiles)[0]);
+      expect(json.files[1]).toEqual(Object.keys(mockFiles)[1]);
+    });
+  });
+
+  describe('storeEtagMetadata', () => {
+    beforeEach(() => {
+      mockFs.restore();
+      mockFs({
+        [rootDir]: {
+          'invalid_techdocs_metadata.json': 'dsds',
+          'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should throw error when the JSON is invalid', async () => {
+      const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json');
+
+      await expect(
+        storeEtagMetadata(filePath, 'etag123abc'),
+      ).rejects.toThrowError('Unexpected token d in JSON at position 0');
+    });
+
+    it('should add etag to the metadata json', async () => {
+      const filePath = path.join(rootDir, 'techdocs_metadata.json');
+
+      await storeEtagMetadata(filePath, 'etag123abc');
+
+      const json = await fs.readJson(filePath);
+      expect(json.etag).toBe('etag123abc');
+    });
+  });
+
+  describe('getMkdocsYml', () => {
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    const inputDir = resolvePath(__filename, '../__fixtures__/');
+
+    it('returns expected contents when .yml file is present', async () => {
+      const key = path.join(inputDir, 'mkdocs.yml');
+      mockFs({ [key]: mkdocsYml });
+      const { path: mkdocsPath, content } = await getMkdocsYml(inputDir);
+
+      expect(mkdocsPath).toBe(key);
+      expect(content).toBe(mkdocsYml.toString());
+    });
+
+    it('returns expected contents when .yaml file is present', async () => {
+      const key = path.join(inputDir, 'mkdocs.yaml');
+      mockFs({ [key]: mkdocsYml });
+      const { path: mkdocsPath, content } = await getMkdocsYml(inputDir);
+      expect(mkdocsPath).toBe(key);
+      expect(content).toBe(mkdocsYml.toString());
+    });
+
+    it('throws when neither .yml nor .yaml file is present', async () => {
+      const invalidInputDir = resolvePath(__filename);
+      await expect(getMkdocsYml(invalidInputDir)).rejects.toThrowError(
+        /Could not read MkDocs YAML config file mkdocs.yml or mkdocs.yaml for validation/,
+      );
+    });
+  });
+
+  describe('validateMkdocsYaml', () => {
+    const inputDir = resolvePath(__filename, '../__fixtures__/');
+
+    it('should return true on when no docs_dir present', async () => {
+      await expect(
+        validateMkdocsYaml(inputDir, mkdocsYml.toString()),
+      ).resolves.toBeUndefined();
+    });
+
+    it('should return true on when a valid docs_dir is present', async () => {
+      await expect(
+        validateMkdocsYaml(inputDir, mkdocsYmlWithValidDocDir.toString()),
+      ).resolves.toBe('docs/');
+    });
+
+    it('should return false on absolute doc_dir path', async () => {
+      await expect(
+        validateMkdocsYaml(inputDir, mkdocsYmlWithInvalidDocDir.toString()),
+      ).rejects.toThrow();
+    });
+
+    it('should return false on doc_dir path that traverses directory structure backwards', async () => {
+      await expect(
+        validateMkdocsYaml(inputDir, mkdocsYmlWithInvalidDocDir2.toString()),
+      ).rejects.toThrow();
+    });
+
+    it('should validate files with custom yaml tags', async () => {
+      await expect(
+        validateMkdocsYaml(inputDir, mkdocsYmlWithExtensions.toString()),
+      ).resolves.toBeUndefined();
+    });
+  });
+});

plugins/techdocs-node/src/stages/generate/helpers.ts

@@ -0,0 +1,318 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { isChildPath } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { assertError, ForwardedError } from '@backstage/errors';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+import { SpawnOptionsWithoutStdio, spawn } from 'child_process';
+import fs from 'fs-extra';
+import gitUrlParse from 'git-url-parse';
+import yaml, { DEFAULT_SCHEMA, Type } from 'js-yaml';
+import path, { resolve as resolvePath } from 'path';
+import { PassThrough, Writable } from 'stream';
+import { Logger } from 'winston';
+import { ParsedLocationAnnotation } from '../../helpers';
+import { SupportedGeneratorKey } from './types';
+import { getFileTreeRecursively } from '../publish/helpers';
+
+// TODO: Implement proper support for more generators.
+export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
+  if (!entity) {
+    throw new Error('No entity provided');
+  }
+
+  return 'techdocs';
+}
+
+export type RunCommandOptions = {
+  /** command to run */
+  command: string;
+  /** arguments to pass the command */
+  args: string[];
+  /** options to pass to spawn */
+  options: SpawnOptionsWithoutStdio;
+  /** stream to capture stdout and stderr output */
+  logStream?: Writable;
+};
+
+/**
+ * Run a command in a sub-process, normally a shell command.
+ */
+export const runCommand = async ({
+  command,
+  args,
+  options,
+  logStream = new PassThrough(),
+}: RunCommandOptions) => {
+  await new Promise<void>((resolve, reject) => {
+    const process = spawn(command, args, options);
+
+    process.stdout.on('data', stream => {
+      logStream.write(stream);
+    });
+
+    process.stderr.on('data', stream => {
+      logStream.write(stream);
+    });
+
+    process.on('error', error => {
+      return reject(error);
+    });
+
+    process.on('close', code => {
+      if (code !== 0) {
+        return reject(`Command ${command} failed, exit code: ${code}`);
+      }
+      return resolve();
+    });
+  });
+};
+
+/**
+ * Return the source url for MkDocs based on the backstage.io/techdocs-ref annotation.
+ * Depending on the type of target, it can either return a repo_url, an edit_uri, both, or none.
+ *
+ * @param parsedLocationAnnotation - Object with location url and type
+ * @param scmIntegrations - the scmIntegration to do url transformations
+ * @param docsFolder - the configured docs folder in the mkdocs.yml (defaults to 'docs')
+ * @returns the settings for the mkdocs.yml
+ */
+export const getRepoUrlFromLocationAnnotation = (
+  parsedLocationAnnotation: ParsedLocationAnnotation,
+  scmIntegrations: ScmIntegrationRegistry,
+  docsFolder: string = 'docs',
+): { repo_url?: string; edit_uri?: string } => {
+  const { type: locationType, target } = parsedLocationAnnotation;
+
+  if (locationType === 'url') {
+    const integration = scmIntegrations.byUrl(target);
+
+    // We only support it for github and gitlab for now as the edit_uri
+    // is not properly supported for others yet.
+    if (integration && ['github', 'gitlab'].includes(integration.type)) {
+      // handle the case where a user manually writes url:https://github.com/backstage/backstage i.e. without /blob/...
+      const { filepathtype } = gitUrlParse(target);
+      if (filepathtype === '') {
+        return { repo_url: target };
+      }
+
+      const sourceFolder = integration.resolveUrl({
+        url: `./${docsFolder}`,
+        base: target,
+      });
+      return { edit_uri: integration.resolveEditUrl(sourceFolder) };
+    }
+  }
+
+  return {};
+};
+
+class UnknownTag {
+  constructor(public readonly data: any, public readonly type?: string) {}
+}
+
+export const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([
+  new Type('', {
+    kind: 'scalar',
+    multi: true,
+    representName: o => (o as UnknownTag).type,
+    represent: o => (o as UnknownTag).data ?? '',
+    instanceOf: UnknownTag,
+    construct: (data: string, type?: string) => new UnknownTag(data, type),
+  }),
+]);
+
+/**
+ * Finds and loads the contents of either an mkdocs.yml or mkdocs.yaml file,
+ * depending on which is present (MkDocs supports both as of v1.2.2).
+ *
+ * @param inputDir - base dir to be searched for either an mkdocs.yml or
+ *   mkdocs.yaml file.
+ */
+export const getMkdocsYml = async (
+  inputDir: string,
+): Promise<{ path: string; content: string }> => {
+  let mkdocsYmlPath: string;
+  let mkdocsYmlFileString: string;
+  try {
+    mkdocsYmlPath = path.join(inputDir, 'mkdocs.yaml');
+    mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8');
+  } catch {
+    try {
+      mkdocsYmlPath = path.join(inputDir, 'mkdocs.yml');
+      mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8');
+    } catch (error) {
+      throw new ForwardedError(
+        'Could not read MkDocs YAML config file mkdocs.yml or mkdocs.yaml for validation',
+        error,
+      );
+    }
+  }
+
+  return {
+    path: mkdocsYmlPath,
+    content: mkdocsYmlFileString,
+  };
+};
+
+/**
+ * Validating mkdocs config file for incorrect/insecure values
+ * Throws on invalid configs
+ *
+ * @param inputDir - base dir to be used as a docs_dir path validity check
+ * @param mkdocsYmlFileString - The string contents of the loaded
+ *   mkdocs.yml or equivalent of a docs site
+ * @returns the parsed docs_dir or undefined
+ */
+export const validateMkdocsYaml = async (
+  inputDir: string,
+  mkdocsYmlFileString: string,
+): Promise<string | undefined> => {
+  const mkdocsYml = yaml.load(mkdocsYmlFileString, {
+    schema: MKDOCS_SCHEMA,
+  });
+
+  if (mkdocsYml === null || typeof mkdocsYml !== 'object') {
+    return undefined;
+  }
+
+  const parsedMkdocsYml: Record<string, any> = mkdocsYml;
+  if (
+    parsedMkdocsYml.docs_dir &&
+    !isChildPath(inputDir, resolvePath(inputDir, parsedMkdocsYml.docs_dir))
+  ) {
+    throw new Error(
+      `docs_dir configuration value in mkdocs can't be an absolute directory or start with ../ for security reasons.
+       Use relative paths instead which are resolved relative to your mkdocs.yml file location.`,
+    );
+  }
+  return parsedMkdocsYml.docs_dir;
+};
+
+/**
+ * Update docs/index.md file before TechDocs generator uses it to generate docs site,
+ * falling back to docs/README.md or README.md in case a default docs/index.md
+ * is not provided.
+ */
+export const patchIndexPreBuild = async ({
+  inputDir,
+  logger,
+  docsDir = 'docs',
+}: {
+  inputDir: string;
+  logger: Logger;
+  docsDir?: string;
+}) => {
+  const docsPath = path.join(inputDir, docsDir);
+  const indexMdPath = path.join(docsPath, 'index.md');
+
+  if (await fs.pathExists(indexMdPath)) {
+    return;
+  }
+  logger.warn(`${path.join(docsDir, 'index.md')} not found.`);
+  const fallbacks = [
+    path.join(docsPath, 'README.md'),
+    path.join(docsPath, 'readme.md'),
+    path.join(inputDir, 'README.md'),
+    path.join(inputDir, 'readme.md'),
+  ];
+
+  await fs.ensureDir(docsPath);
+  for (const filePath of fallbacks) {
+    try {
+      await fs.copyFile(filePath, indexMdPath);
+      return;
+    } catch (error) {
+      logger.warn(`${path.relative(inputDir, filePath)} not found.`);
+    }
+  }
+
+  logger.warn(
+    `Could not find any techdocs' index file. Please make sure at least one of ${[
+      indexMdPath,
+      ...fallbacks,
+    ].join(' ')} exists.`,
+  );
+};
+
+/**
+ * Create or update the techdocs_metadata.json. Values initialized/updated are:
+ * - The build_timestamp (now)
+ * - The list of files generated
+ *
+ * @param techdocsMetadataPath - File path to techdocs_metadata.json
+ */
+export const createOrUpdateMetadata = async (
+  techdocsMetadataPath: string,
+  logger: Logger,
+): Promise<void> => {
+  const techdocsMetadataDir = techdocsMetadataPath
+    .split(path.sep)
+    .slice(0, -1)
+    .join(path.sep);
+  // check if file exists, create if it does not.
+  try {
+    await fs.access(techdocsMetadataPath, fs.constants.F_OK);
+  } catch (err) {
+    // Bootstrap file with empty JSON
+    await fs.writeJson(techdocsMetadataPath, JSON.parse('{}'));
+  }
+  // check if valid Json
+  let json;
+  try {
+    json = await fs.readJson(techdocsMetadataPath);
+  } catch (err) {
+    assertError(err);
+    const message = `Invalid JSON at ${techdocsMetadataPath} with error ${err.message}`;
+    logger.error(message);
+    throw new Error(message);
+  }
+
+  json.build_timestamp = Date.now();
+
+  // Get and write generated files to the metadata JSON. Each file string is in
+  // a form appropriate for invalidating the associated object from cache.
+  try {
+    json.files = (await getFileTreeRecursively(techdocsMetadataDir)).map(file =>
+      file.replace(`${techdocsMetadataDir}${path.sep}`, ''),
+    );
+  } catch (err) {
+    assertError(err);
+    json.files = [];
+    logger.warn(`Unable to add files list to metadata: ${err.message}`);
+  }
+
+  await fs.writeJson(techdocsMetadataPath, json);
+  return;
+};
+
+/**
+ * Update the techdocs_metadata.json to add etag of the prepared tree (e.g. commit SHA or actual Etag of the resource).
+ * This is helpful to check if a TechDocs site in storage has gone outdated, without maintaining an in-memory build info
+ * per Backstage instance.
+ *
+ * @param techdocsMetadataPath - File path to techdocs_metadata.json
+ * @param etag - The ETag to use
+ */
+export const storeEtagMetadata = async (
+  techdocsMetadataPath: string,
+  etag: string,
+): Promise<void> => {
+  const json = await fs.readJson(techdocsMetadataPath);
+  json.etag = etag;
+  await fs.writeJson(techdocsMetadataPath, json);
+};

plugins/techdocs-node/src/stages/generate/index.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { TechdocsGenerator } from './techdocs';
+export { Generators } from './generators';
+export type {
+  GeneratorBase,
+  GeneratorOptions,
+  GeneratorBuilder,
+  GeneratorRunOptions,
+  SupportedGeneratorKey,
+} from './types';

plugins/techdocs-node/src/stages/generate/mkDocsPatchers.ts

@@ -0,0 +1,166 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Logger } from 'winston';
+import fs from 'fs-extra';
+import yaml from 'js-yaml';
+import { ParsedLocationAnnotation } from '../../helpers';
+import { getRepoUrlFromLocationAnnotation, MKDOCS_SCHEMA } from './helpers';
+import { assertError } from '@backstage/errors';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+
+type MkDocsObject = {
+  plugins?: string[];
+  docs_dir: string;
+  repo_url?: string;
+  edit_uri?: string;
+};
+
+const patchMkdocsFile = async (
+  mkdocsYmlPath: string,
+  logger: Logger,
+  updateAction: (mkdocsYml: MkDocsObject) => boolean,
+) => {
+  // We only want to override the mkdocs.yml if it has actually changed. This is relevant if
+  // used with a 'dir' location on the file system as this would permanently update the file.
+  let didEdit = false;
+
+  let mkdocsYmlFileString;
+  try {
+    mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8');
+  } catch (error) {
+    assertError(error);
+    logger.warn(
+      `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`,
+    );
+    return;
+  }
+
+  let mkdocsYml: any;
+  try {
+    mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA });
+
+    // mkdocsYml should be an object type after successful parsing.
+    // But based on its type definition, it can also be a string or undefined, which we don't want.
+    if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') {
+      throw new Error('Bad YAML format.');
+    }
+  } catch (error) {
+    assertError(error);
+    logger.warn(
+      `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`,
+    );
+    return;
+  }
+
+  didEdit = updateAction(mkdocsYml);
+
+  try {
+    if (didEdit) {
+      await fs.writeFile(
+        mkdocsYmlPath,
+        yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }),
+        'utf8',
+      );
+    }
+  } catch (error) {
+    assertError(error);
+    logger.warn(
+      `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`,
+    );
+    return;
+  }
+};
+
+/**
+ * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site.
+ *
+ * List of tasks:
+ * - Add repo_url or edit_uri if it does not exists
+ * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default.
+ * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get
+ * the repository URL.
+ *
+ * This function will not throw an error since this is not critical to the whole TechDocs pipeline.
+ * Instead it will log warnings if there are any errors in reading, parsing or writing YAML.
+ *
+ * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site
+ * @param logger - A logger instance
+ * @param parsedLocationAnnotation - Object with location url and type
+ * @param scmIntegrations - the scmIntegration to do url transformations
+ */
+export const patchMkdocsYmlPreBuild = async (
+  mkdocsYmlPath: string,
+  logger: Logger,
+  parsedLocationAnnotation: ParsedLocationAnnotation,
+  scmIntegrations: ScmIntegrationRegistry,
+) => {
+  await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => {
+    if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) {
+      // Add edit_uri and/or repo_url to mkdocs.yml if it is missing.
+      // This will enable the Page edit button generated by MkDocs.
+      // If the either has been set, keep the original value
+      const result = getRepoUrlFromLocationAnnotation(
+        parsedLocationAnnotation,
+        scmIntegrations,
+        mkdocsYml.docs_dir,
+      );
+
+      if (result.repo_url || result.edit_uri) {
+        mkdocsYml.repo_url = result.repo_url;
+        mkdocsYml.edit_uri = result.edit_uri;
+
+        logger.info(
+          `Set ${JSON.stringify(
+            result,
+          )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`,
+        );
+        return true;
+      }
+    }
+    return false;
+  });
+};
+
+/**
+ * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site.
+ *
+ * List of tasks:
+ * - Add techdocs-core plugin to mkdocs file if it doesn't exist
+ *
+ * This function will not throw an error since this is not critical to the whole TechDocs pipeline.
+ * Instead it will log warnings if there are any errors in reading, parsing or writing YAML.
+ *
+ * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site
+ * @param logger - A logger instance
+ */
+export const pathMkdocsYmlWithTechdocsPlugin = async (
+  mkdocsYmlPath: string,
+  logger: Logger,
+) => {
+  await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => {
+    // Modify mkdocs.yaml to contain the needed techdocs-core plugin if it is not there
+    if (!('plugins' in mkdocsYml)) {
+      mkdocsYml.plugins = ['techdocs-core'];
+      return true;
+    }
+
+    if (mkdocsYml.plugins && !mkdocsYml.plugins.includes('techdocs-core')) {
+      mkdocsYml.plugins.push('techdocs-core');
+      return true;
+    }
+    return false;
+  });
+};

plugins/techdocs-node/src/stages/generate/techdocs.test.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readGeneratorConfig } from './techdocs';
+
+const mockLogger = {
+  warn: jest.fn(),
+};
+
+describe('readGeneratorConfig', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  const logger = mockLogger as any;
+
+  it('defaults to runIn docker', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {},
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: undefined,
+      pullImage: undefined,
+    });
+  });
+
+  it('should read local config', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'local',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'local',
+    });
+  });
+
+  it('should read docker config', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+    });
+  });
+
+  it('should read custom docker image', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+          dockerImage: 'my-org/techdocs',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: 'my-org/techdocs',
+    });
+  });
+
+  it('should read config disabling docker pull', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+          dockerImage: 'my-org/techdocs',
+          pullImage: false,
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: 'my-org/techdocs',
+      pullImage: false,
+    });
+  });
+
+  describe('with legacy techdocs.generators.techdocs config', () => {
+    it('should read legacy docker option', () => {
+      const config = new ConfigReader({
+        techdocs: {
+          generators: {
+            techdocs: 'docker',
+          },
+        },
+      });
+
+      expect(readGeneratorConfig(config, logger)).toEqual({
+        runIn: 'docker',
+      });
+    });
+
+    it('legacy option should log warning', () => {
+      const config = new ConfigReader({
+        techdocs: {
+          generators: {
+            techdocs: 'local',
+          },
+        },
+      });
+
+      expect(readGeneratorConfig(config, logger)).toEqual({
+        runIn: 'local',
+      });
+      expect(logger.warn).toHaveBeenCalledWith(
+        `The 'techdocs.generators.techdocs' configuration key is deprecated and will be removed in the future. Please use 'techdocs.generator' instead. ` +
+          `See here https://backstage.io/docs/features/techdocs/configuration`,
+      );
+    });
+  });
+});

plugins/techdocs-node/src/stages/generate/techdocs.ts

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ContainerRunner } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+import path from 'path';
+import { Logger } from 'winston';
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
+import {
+  createOrUpdateMetadata,
+  getMkdocsYml,
+  patchIndexPreBuild,
+  runCommand,
+  storeEtagMetadata,
+  validateMkdocsYaml,
+} from './helpers';
+
+import {
+  patchMkdocsYmlPreBuild,
+  pathMkdocsYmlWithTechdocsPlugin,
+} from './mkDocsPatchers';
+import {
+  GeneratorBase,
+  GeneratorConfig,
+  GeneratorOptions,
+  GeneratorRunInType,
+  GeneratorRunOptions,
+} from './types';
+import { ForwardedError } from '@backstage/errors';
+
+/**
+ * Generates documentation files
+ * @public
+ */
+export class TechdocsGenerator implements GeneratorBase {
+  /**
+   * The default docker image (and version) used to generate content. Public
+   * and static so that techdocs-common consumers can use the same version.
+   */
+  public static readonly defaultDockerImage = 'spotify/techdocs:v0.3.7';
+  private readonly logger: Logger;
+  private readonly containerRunner: ContainerRunner;
+  private readonly options: GeneratorConfig;
+  private readonly scmIntegrations: ScmIntegrationRegistry;
+
+  /**
+   * Returns a instance of TechDocs generator
+   * @param config - A Backstage configuration
+   * @param options - Options to configure the generator
+   */
+  static fromConfig(config: Config, options: GeneratorOptions) {
+    const { containerRunner, logger } = options;
+    const scmIntegrations = ScmIntegrations.fromConfig(config);
+    return new TechdocsGenerator({
+      logger,
+      containerRunner,
+      config,
+      scmIntegrations,
+    });
+  }
+
+  constructor(options: {
+    logger: Logger;
+    containerRunner: ContainerRunner;
+    config: Config;
+    scmIntegrations: ScmIntegrationRegistry;
+  }) {
+    this.logger = options.logger;
+    this.options = readGeneratorConfig(options.config, options.logger);
+    this.containerRunner = options.containerRunner;
+    this.scmIntegrations = options.scmIntegrations;
+  }
+
+  /** {@inheritDoc GeneratorBase.run} */
+  public async run(options: GeneratorRunOptions): Promise<void> {
+    const {
+      inputDir,
+      outputDir,
+      parsedLocationAnnotation,
+      etag,
+      logger: childLogger,
+      logStream,
+    } = options;
+
+    // Do some updates to mkdocs.yml before generating docs e.g. adding repo_url
+    const { path: mkdocsYmlPath, content } = await getMkdocsYml(inputDir);
+
+    // validate the docs_dir first
+    const docsDir = await validateMkdocsYaml(inputDir, content);
+
+    if (parsedLocationAnnotation) {
+      await patchMkdocsYmlPreBuild(
+        mkdocsYmlPath,
+        childLogger,
+        parsedLocationAnnotation,
+        this.scmIntegrations,
+      );
+      await patchIndexPreBuild({ inputDir, logger: childLogger, docsDir });
+    }
+
+    if (!this.options.omitTechdocsCoreMkdocsPlugin) {
+      await pathMkdocsYmlWithTechdocsPlugin(mkdocsYmlPath, childLogger);
+    }
+
+    // Directories to bind on container
+    const mountDirs = {
+      [inputDir]: '/input',
+      [outputDir]: '/output',
+    };
+
+    try {
+      switch (this.options.runIn) {
+        case 'local':
+          await runCommand({
+            command: 'mkdocs',
+            args: ['build', '-d', outputDir, '-v'],
+            options: {
+              cwd: inputDir,
+            },
+            logStream,
+          });
+          childLogger.info(
+            `Successfully generated docs from ${inputDir} into ${outputDir} using local mkdocs`,
+          );
+          break;
+        case 'docker':
+          await this.containerRunner.runContainer({
+            imageName:
+              this.options.dockerImage ?? TechdocsGenerator.defaultDockerImage,
+            args: ['build', '-d', '/output'],
+            logStream,
+            mountDirs,
+            workingDir: '/input',
+            // Set the home directory inside the container as something that applications can
+            // write to, otherwise they will just fail trying to write to /
+            envVars: { HOME: '/tmp' },
+            pullImage: this.options.pullImage,
+          });
+          childLogger.info(
+            `Successfully generated docs from ${inputDir} into ${outputDir} using techdocs-container`,
+          );
+          break;
+        default:
+          throw new Error(
+            `Invalid config value "${this.options.runIn}" provided in 'techdocs.generators.techdocs'.`,
+          );
+      }
+    } catch (error) {
+      this.logger.debug(
+        `Failed to generate docs from ${inputDir} into ${outputDir}`,
+      );
+      throw new ForwardedError(
+        `Failed to generate docs from ${inputDir} into ${outputDir}`,
+        error,
+      );
+    }
+
+    /**
+     * Post Generate steps
+     */
+
+    // Add build timestamp and files to techdocs_metadata.json
+    // Creates techdocs_metadata.json if file does not exist.
+    await createOrUpdateMetadata(
+      path.join(outputDir, 'techdocs_metadata.json'),
+      childLogger,
+    );
+
+    // Add etag of the prepared tree to techdocs_metadata.json
+    // Assumes that the file already exists.
+    if (etag) {
+      await storeEtagMetadata(
+        path.join(outputDir, 'techdocs_metadata.json'),
+        etag,
+      );
+    }
+  }
+}
+
+export function readGeneratorConfig(
+  config: Config,
+  logger: Logger,
+): GeneratorConfig {
+  const legacyGeneratorType = config.getOptionalString(
+    'techdocs.generators.techdocs',
+  ) as GeneratorRunInType;
+
+  if (legacyGeneratorType) {
+    logger.warn(
+      `The 'techdocs.generators.techdocs' configuration key is deprecated and will be removed in the future. Please use 'techdocs.generator' instead. ` +
+        `See here https://backstage.io/docs/features/techdocs/configuration`,
+    );
+  }
+
+  return {
+    runIn:
+      legacyGeneratorType ??
+      config.getOptionalString('techdocs.generator.runIn') ??
+      'docker',
+    dockerImage: config.getOptionalString('techdocs.generator.dockerImage'),
+    pullImage: config.getOptionalBoolean('techdocs.generator.pullImage'),
+    omitTechdocsCoreMkdocsPlugin: config.getOptionalBoolean(
+      'techdocs.generator.mkdocs.omitTechdocsCorePlugin',
+    ),
+  };
+}

plugins/techdocs-node/src/stages/generate/types.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ContainerRunner } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { Writable } from 'stream';
+import { Logger } from 'winston';
+import { ParsedLocationAnnotation } from '../../helpers';
+
+// Determines where the generator will be run
+export type GeneratorRunInType = 'docker' | 'local';
+
+/**
+ * Options for building generators
+ * @public
+ */
+export type GeneratorOptions = {
+  containerRunner: ContainerRunner;
+  logger: Logger;
+};
+
+/**
+ * The techdocs generator configurations options.
+ */
+export type GeneratorConfig = {
+  runIn: GeneratorRunInType;
+  dockerImage?: string;
+  pullImage?: boolean;
+  omitTechdocsCoreMkdocsPlugin?: boolean;
+};
+
+/**
+ * The values that the generator will receive.
+ *
+ * @public
+ * @param inputDir - The directory of the uncompiled documentation, with the values from the frontend
+ * @param outputDir - Directory to store generated docs in. Usually - a newly created temporary directory.
+ * @param parsedLocationAnnotation - backstage.io/techdocs-ref annotation of an entity
+ * @param etag - A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json.
+ * @param logger - A logger that forwards the messages to the caller to be displayed outside of the backend.
+ * @param logStream - A log stream that can send raw log messages to the caller to be displayed outside of the backend.
+ */
+export type GeneratorRunOptions = {
+  inputDir: string;
+  outputDir: string;
+  parsedLocationAnnotation?: ParsedLocationAnnotation;
+  etag?: string;
+  logger: Logger;
+  logStream?: Writable;
+};
+
+/**
+ * Generates documentation files
+ * @public
+ */
+export type GeneratorBase = {
+  /**
+   * Runs the generator with the values
+   * @public
+   */
+  run(opts: GeneratorRunOptions): Promise<void>;
+};
+
+/**
+ * List of supported generator options
+ * @public
+ */
+export type SupportedGeneratorKey = 'techdocs' | string;
+
+/**
+ * The generator builder holds the generator ready for run time
+ * @public
+ */
+export type GeneratorBuilder = {
+  register(protocol: SupportedGeneratorKey, generator: GeneratorBase): void;
+  get(entity: Entity): GeneratorBase;
+};

plugins/techdocs-node/src/stages/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './generate';
+export * from './prepare';
+export * from './publish';

plugins/techdocs-node/src/stages/prepare/dir.test.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger, UrlReader } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { DirectoryPreparer } from './dir';
+
+function normalizePath(path: string) {
+  return path
+    .replace(/^[a-z]:/i, '')
+    .split('\\')
+    .join('/');
+}
+
+jest.mock('../../helpers', () => ({
+  ...jest.requireActual<{}>('../../helpers'),
+}));
+
+const logger = getVoidLogger();
+
+const createMockEntity = (annotations: {}) => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'testName',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const mockConfig = new ConfigReader({});
+const mockUrlReader: jest.Mocked<UrlReader> = {
+  read: jest.fn(),
+  readTree: jest.fn(),
+  search: jest.fn(),
+};
+
+describe('directory preparer', () => {
+  it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
+
+    const mockEntity = createMockEntity({
+      'backstage.io/managed-by-location':
+        'file:/directory/documented-component.yaml',
+      'backstage.io/techdocs-ref': 'dir:./our-documentation',
+    });
+
+    const { preparedDir } = await directoryPreparer.prepare(mockEntity);
+    expect(normalizePath(preparedDir)).toEqual('/directory/our-documentation');
+  });
+
+  it('should reject when techdocs-ref is absolute', async () => {
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
+
+    const mockEntity = createMockEntity({
+      'backstage.io/managed-by-location':
+        'file:/directory/documented-component.yaml',
+      'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
+    });
+
+    await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
+      /Relative path is not allowed to refer to a directory outside its parent/,
+    );
+  });
+
+  it('should reject when managed-by-location has an unknown type', async () => {
+    const directoryPreparer = new DirectoryPreparer(
+      mockConfig,
+      logger,
+      mockUrlReader,
+    );
+
+    const mockEntity = createMockEntity({
+      'backstage.io/managed-by-location':
+        'does-not-exist:https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+      'backstage.io/techdocs-ref': 'dir:./docs',
+    });
+
+    await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
+      /Unable to resolve location type does-not-exist/,
+    );
+  });
+});

plugins/techdocs-node/src/stages/prepare/dir.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { InputError } from '@backstage/errors';
+import {
+  ScmIntegrationRegistry,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { Logger } from 'winston';
+import { parseReferenceAnnotation, transformDirLocation } from '../../helpers';
+import {
+  PreparerBase,
+  PreparerConfig,
+  PreparerOptions,
+  PreparerResponse,
+} from './types';
+
+/**
+ * Preparer used to retrieve documentation files from a local directory
+ * @public
+ */
+export class DirectoryPreparer implements PreparerBase {
+  private readonly scmIntegrations: ScmIntegrationRegistry;
+  private readonly reader: UrlReader;
+
+  /**  @deprecated use static fromConfig method instead */
+  constructor(config: Config, _logger: Logger | null, reader: UrlReader) {
+    this.reader = reader;
+    this.scmIntegrations = ScmIntegrations.fromConfig(config);
+  }
+
+  /**
+   * Returns a directory preparer instance
+   * @param config - A backstage config
+   * @param options - A directory preparer options containing a logger and reader
+   */
+  static fromConfig(
+    config: Config,
+    { logger, reader }: PreparerConfig,
+  ): DirectoryPreparer {
+    return new DirectoryPreparer(config, logger, reader);
+  }
+
+  /** {@inheritDoc PreparerBase.prepare} */
+  async prepare(
+    entity: Entity,
+    options?: PreparerOptions,
+  ): Promise<PreparerResponse> {
+    const annotation = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+    const { type, target } = transformDirLocation(
+      entity,
+      annotation,
+      this.scmIntegrations,
+    );
+
+    switch (type) {
+      case 'url': {
+        options?.logger?.debug(`Reading files from ${target}`);
+        // the target is an absolute url since it has already been transformed
+        const response = await this.reader.readTree(target, {
+          etag: options?.etag,
+        });
+        const preparedDir = await response.dir();
+
+        options?.logger?.debug(`Tree downloaded and stored at ${preparedDir}`);
+
+        return {
+          preparedDir,
+          etag: response.etag,
+        };
+      }
+
+      case 'dir': {
+        return {
+          // the transformation already validated that the target is in a safe location
+          preparedDir: target,
+          // Instead of supporting caching on local sources, use techdocs-cli for local development and debugging.
+          etag: '',
+        };
+      }
+
+      default:
+        throw new InputError(`Unable to resolve location type ${type}`);
+    }
+  }
+}

plugins/techdocs-node/src/stages/prepare/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { DirectoryPreparer } from './dir';
+export { UrlPreparer } from './url';
+export { Preparers } from './preparers';
+export type {
+  PreparerBase,
+  PreparerBuilder,
+  PreparerConfig,
+  PreparerOptions,
+  PreparerResponse,
+  RemoteProtocol,
+  ETag,
+} from './types';

plugins/techdocs-node/src/stages/prepare/preparers.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { parseReferenceAnnotation } from '../../helpers';
+import { DirectoryPreparer } from './dir';
+import { UrlPreparer } from './url';
+import {
+  PreparerBase,
+  PreparerBuilder,
+  PreparerConfig,
+  RemoteProtocol,
+} from './types';
+
+/**
+ * Collection of docs preparers (dir and url)
+ * @public
+ */
+export class Preparers implements PreparerBuilder {
+  private preparerMap = new Map<RemoteProtocol, PreparerBase>();
+
+  /**
+   * Returns a generators instance containing a generator for TechDocs
+   * @public
+   * @param backstageConfig - A Backstage configuration
+   * @param preparerConfig - Options to configure preparers
+   */
+  static async fromConfig(
+    backstageConfig: Config,
+    { logger, reader }: PreparerConfig,
+  ): Promise<PreparerBuilder> {
+    const preparers = new Preparers();
+
+    const urlPreparer = new UrlPreparer(reader, logger);
+    preparers.register('url', urlPreparer);
+
+    /**
+     * Dir preparer is a syntactic sugar for users to define techdocs-ref annotation.
+     * When using dir preparer, the docs will be fetched using URL Reader.
+     */
+    const directoryPreparer = new DirectoryPreparer(
+      backstageConfig,
+      logger,
+      reader,
+    );
+    preparers.register('dir', directoryPreparer);
+
+    return preparers;
+  }
+
+  /**
+   * Register a preparer in the preparers collection
+   * @param protocol - url or dir to associate with preparer
+   * @param preparer - The preparer instance to set
+   */
+  register(protocol: RemoteProtocol, preparer: PreparerBase) {
+    this.preparerMap.set(protocol, preparer);
+  }
+
+  /**
+   * Returns the preparer for a given TechDocs entity
+   * @param entity - A TechDocs entity instance
+   * @returns
+   */
+  get(entity: Entity): PreparerBase {
+    const { type } = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+    const preparer = this.preparerMap.get(type);
+
+    if (!preparer) {
+      throw new Error(`No preparer registered for type: "${type}"`);
+    }
+
+    return preparer;
+  }
+}

plugins/techdocs-node/src/stages/prepare/types.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Entity } from '@backstage/catalog-model';
+import { UrlReader } from '@backstage/backend-common';
+import { Logger } from 'winston';
+
+/**
+ * A unique identifier of the tree blob, usually the commit SHA or etag from the target.
+ * @public
+ */
+export type ETag = string;
+
+/**
+ * Options for building preparers
+ * @public
+ */
+export type PreparerConfig = {
+  logger: Logger;
+  reader: UrlReader;
+};
+
+/**
+ * Options for configuring the content preparation process.
+ * @public
+ */
+export type PreparerOptions = {
+  /**
+   * An instance of the logger
+   */
+  logger?: Logger;
+  /**
+   * see {@link ETag}
+   */
+  etag?: ETag;
+};
+
+/**
+ * Result of the preparation step.
+ * @public
+ */
+export type PreparerResponse = {
+  /**
+   * The path to directory where the tree is downloaded.
+   */
+  preparedDir: string;
+  /**
+   * see {@link ETag}
+   */
+  etag: ETag;
+};
+
+/**
+ * Definition of a TechDocs preparer
+ * @public
+ */
+export type PreparerBase = {
+  /**
+   * Given an Entity definition from the Software Catalog, go and prepare a directory
+   * with contents from the location in temporary storage and return the path.
+   *
+   * @param entity - The entity from the Software Catalog
+   * @param options - If etag is provided, it will be used to check if the target has
+   *        updated since the last build.
+   * @throws `NotModifiedError` when the prepared directory has not been changed since the last build.
+   */
+  prepare(entity: Entity, options?: PreparerOptions): Promise<PreparerResponse>;
+};
+
+/**
+ * Definition for a TechDocs preparer builder
+ * @public
+ */
+export type PreparerBuilder = {
+  register(protocol: RemoteProtocol, preparer: PreparerBase): void;
+  get(entity: Entity): PreparerBase;
+};
+
+/**
+ * Location where documentation files are stored
+ * @public
+ */
+export type RemoteProtocol = 'url' | 'dir';

plugins/techdocs-node/src/stages/prepare/url.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { assertError } from '@backstage/errors';
+import { UrlReader } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { Logger } from 'winston';
+import { getDocFilesFromRepository } from '../../helpers';
+import {
+  PreparerBase,
+  PreparerConfig,
+  PreparerOptions,
+  PreparerResponse,
+} from './types';
+
+/**
+ * Preparer used to retrieve documentation files from a remote repository
+ * @public
+ */
+export class UrlPreparer implements PreparerBase {
+  private readonly logger: Logger;
+  private readonly reader: UrlReader;
+
+  /**  @deprecated use static fromConfig method instead */
+  constructor(reader: UrlReader, logger: Logger) {
+    this.logger = logger;
+    this.reader = reader;
+  }
+
+  /**
+   * Returns a directory preparer instance
+   * @param config - A URL preparer config containing the a logger and reader
+   */
+  static fromConfig({ reader, logger }: PreparerConfig): UrlPreparer {
+    return new UrlPreparer(reader, logger);
+  }
+
+  /** {@inheritDoc PreparerBase.prepare} */
+  async prepare(
+    entity: Entity,
+    options?: PreparerOptions,
+  ): Promise<PreparerResponse> {
+    try {
+      return await getDocFilesFromRepository(this.reader, entity, {
+        etag: options?.etag,
+        logger: this.logger,
+      });
+    } catch (error) {
+      assertError(error);
+      // NotModifiedError means that etag based cache is still valid.
+      if (error.name === 'NotModifiedError') {
+        this.logger.debug(`Cache is valid for etag ${options?.etag}`);
+      } else {
+        this.logger.debug(
+          `Unable to fetch files for building docs ${error.message}`,
+        );
+      }
+
+      throw error;
+    }
+  }
+}

plugins/techdocs-node/src/stages/publish/awsS3.test.ts

@@ -0,0 +1,502 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import express from 'express';
+import request from 'supertest';
+import mockFs from 'mock-fs';
+import path from 'path';
+import fs from 'fs-extra';
+import { AwsS3Publish } from './awsS3';
+
+// NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library
+
+const rootDir = (global as any).rootDir; // Set by setupTests.ts
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.join(rootDir, namespace || DEFAULT_NAMESPACE, kind, name);
+};
+
+const logger = getVoidLogger();
+const loggerInfoSpy = jest.spyOn(logger, 'info');
+const loggerErrorSpy = jest.spyOn(logger, 'error');
+
+const createPublisherFromConfig = ({
+  bucketName = 'bucketName',
+  bucketRootPath = '/',
+  legacyUseCaseSensitiveTripletPaths = false,
+  sse,
+}: {
+  bucketName?: string;
+  bucketRootPath?: string;
+  legacyUseCaseSensitiveTripletPaths?: boolean;
+  sse?: string;
+} = {}) => {
+  const mockConfig = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7007',
+      publisher: {
+        type: 'awsS3',
+        awsS3: {
+          credentials: {
+            accessKeyId: 'accessKeyId',
+            secretAccessKey: 'secretAccessKey',
+          },
+          bucketName,
+          bucketRootPath,
+          sse,
+        },
+      },
+      legacyUseCaseSensitiveTripletPaths,
+    },
+  });
+
+  return AwsS3Publish.fromConfig(mockConfig, logger);
+};
+
+describe('AwsS3Publish', () => {
+  const entity = {
+    apiVersion: '1',
+    kind: 'Component',
+    metadata: {
+      name: 'backstage',
+      namespace: 'default',
+      annotations: {},
+    },
+  };
+
+  const entityName = {
+    kind: 'Component',
+    name: 'backstage',
+    namespace: 'default',
+  };
+
+  const techdocsMetadata = {
+    site_name: 'backstage',
+    site_description: 'site_content',
+    etag: 'etag',
+    build_timestamp: 612741599,
+  };
+
+  const directory = getEntityRootDir(entity);
+
+  const files = {
+    'index.html': '',
+    '404.html': '',
+    'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
+    assets: {
+      'main.css': '',
+    },
+    html: {
+      'unsafe.html': '<html></html>',
+    },
+    img: {
+      'with spaces.png': 'found it',
+      'unsafe.svg': '<svg></svg>',
+    },
+    'some folder': {
+      'also with spaces.js': 'found it too',
+    },
+  };
+
+  beforeEach(() => {
+    mockFs({
+      [directory]: files,
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  describe('getReadiness', () => {
+    it('should validate correct config', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketName: 'errorBucket',
+      });
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: false,
+      });
+    });
+  });
+
+  describe('publish', () => {
+    it('should publish a directory', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory as well when legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory when root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/component/backstage/404.html',
+          `backstage-data/techdocs/default/component/backstage/index.html`,
+          `backstage-data/techdocs/default/component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory when root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/Component/backstage/404.html',
+          `backstage-data/techdocs/default/Component/backstage/index.html`,
+          `backstage-data/techdocs/default/Component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory when sse is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        sse: 'aws:kms',
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          'default/component/backstage/index.html',
+          'default/component/backstage/assets/main.css',
+        ]),
+      });
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = path.join(
+        rootDir,
+        'wrong',
+        'path',
+        'to',
+        'generatedDirectory',
+      );
+
+      const publisher = createPublisherFromConfig();
+
+      const fails = publisher.publish({
+        entity,
+        directory: wrongPathToGeneratedDirectory,
+      });
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(
+          'Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory',
+        ),
+      });
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(wrongPathToGeneratedDirectory),
+      });
+    });
+
+    it('should delete stale files after upload', async () => {
+      const bucketName = 'delete_stale_files_success';
+      const publisher = createPublisherFromConfig({ bucketName: bucketName });
+      await publisher.publish({ entity, directory });
+      expect(loggerInfoSpy).toHaveBeenLastCalledWith(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`,
+      );
+    });
+
+    it('should log error when the stale files deletion fails', async () => {
+      const bucketName = 'delete_stale_files_error';
+      const publisher = createPublisherFromConfig({ bucketName: bucketName });
+      await publisher.publish({ entity, directory });
+      expect(loggerErrorSpy).toHaveBeenLastCalledWith(
+        'Unable to delete file(s) from AWS S3. Error: Message',
+      );
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(
+        await publisher.hasDocsBeenGenerated({
+          kind: 'entity',
+          metadata: {
+            namespace: 'invalid',
+            name: 'triplet',
+          },
+        } as Entity),
+      ).toBe(false);
+    });
+  });
+
+  describe('fetchTechDocsMetadata', () => {
+    it('should return tech docs metadata', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata even if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata when json encoded with single quotes', async () => {
+      const techdocsMetadataPath = path.join(
+        directory,
+        'techdocs_metadata.json',
+      );
+      const techdocsMetadataContent = files['techdocs_metadata.json'];
+
+      fs.writeFileSync(
+        techdocsMetadataPath,
+        techdocsMetadataContent.replace(/"/g, "'"),
+      );
+
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+
+      fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent);
+    });
+
+    it('should return an error if the techdocs_metadata.json file is not present', async () => {
+      const publisher = createPublisherFromConfig();
+
+      const invalidEntityName = {
+        namespace: 'invalid',
+        kind: 'triplet',
+        name: 'path',
+      };
+
+      const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringMatching(
+          /TechDocs metadata fetch failed; caused by Error: The file .* does not exist/i,
+        ),
+      });
+    });
+  });
+
+  describe('docsRouter', () => {
+    const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+
+    let app: express.Express;
+
+    beforeEach(async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+    });
+
+    it('should pass expected object path to bucket', async () => {
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass text/plain content-type for html', async () => {
+      const htmlResponse = await request(app).get(
+        `/${entityTripletPath}/html/unsafe.html`,
+      );
+      expect(htmlResponse.text).toEqual('<html></html>');
+      expect(htmlResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+
+      const svgResponse = await request(app).get(
+        `/${entityTripletPath}/img/unsafe.svg`,
+      );
+      expect(svgResponse.text).toEqual('<svg></svg>');
+      expect(svgResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    });
+
+    it('should return 404 if file is not found', async () => {
+      const response = await request(app).get(
+        `/${entityTripletPath}/not-found.html`,
+      );
+      expect(response.status).toBe(404);
+
+      expect(Buffer.from(response.text).toString('utf8')).toEqual(
+        'File Not Found',
+      );
+    });
+  });
+});

plugins/techdocs-node/src/stages/publish/awsS3.ts

@@ -0,0 +1,514 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { assertError, ForwardedError } from '@backstage/errors';
+import aws, { Credentials } from 'aws-sdk';
+import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
+import { CredentialsOptions } from 'aws-sdk/lib/credentials';
+import express from 'express';
+import fs from 'fs-extra';
+import JSON5 from 'json5';
+import createLimiter from 'p-limit';
+import path from 'path';
+import { Readable } from 'stream';
+import { Logger } from 'winston';
+import {
+  bulkStorageOperation,
+  getCloudPathForLocalPath,
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  getStaleFiles,
+  lowerCaseEntityTriplet,
+  lowerCaseEntityTripletInStoragePath,
+  normalizeExternalStorageRootPath,
+} from './helpers';
+import {
+  PublisherBase,
+  PublishRequest,
+  PublishResponse,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';
+
+const streamToBuffer = (stream: Readable): Promise<Buffer> => {
+  return new Promise((resolve, reject) => {
+    try {
+      const chunks: any[] = [];
+      stream.on('data', chunk => chunks.push(chunk));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(chunks)));
+    } catch (e) {
+      throw new ForwardedError('Unable to parse the response data', e);
+    }
+  });
+};
+
+export class AwsS3Publish implements PublisherBase {
+  private readonly storageClient: aws.S3;
+  private readonly bucketName: string;
+  private readonly legacyPathCasing: boolean;
+  private readonly logger: Logger;
+  private readonly bucketRootPath: string;
+  private readonly sse?: 'aws:kms' | 'AES256';
+
+  constructor(options: {
+    storageClient: aws.S3;
+    bucketName: string;
+    legacyPathCasing: boolean;
+    logger: Logger;
+    bucketRootPath: string;
+    sse?: 'aws:kms' | 'AES256';
+  }) {
+    this.storageClient = options.storageClient;
+    this.bucketName = options.bucketName;
+    this.legacyPathCasing = options.legacyPathCasing;
+    this.logger = options.logger;
+    this.bucketRootPath = options.bucketRootPath;
+    this.sse = options.sse;
+  }
+
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
+    let bucketName = '';
+    try {
+      bucketName = config.getString('techdocs.publisher.awsS3.bucketName');
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'awsS3' in your app config, " +
+          'techdocs.publisher.awsS3.bucketName is required.',
+      );
+    }
+
+    const bucketRootPath = normalizeExternalStorageRootPath(
+      config.getOptionalString('techdocs.publisher.awsS3.bucketRootPath') || '',
+    );
+
+    const sse = config.getOptionalString('techdocs.publisher.awsS3.sse') as
+      | 'aws:kms'
+      | 'AES256'
+      | undefined;
+
+    // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
+    // 1. AWS environment variables
+    // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
+    // 2. AWS shared credentials file at ~/.aws/credentials
+    // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html
+    // 3. IAM Roles for EC2
+    // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html
+    const credentialsConfig = config.getOptionalConfig(
+      'techdocs.publisher.awsS3.credentials',
+    );
+    const credentials = AwsS3Publish.buildCredentials(credentialsConfig);
+
+    // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
+    // or AWS shared credentials file at ~/.aws/credentials will be used.
+    const region = config.getOptionalString('techdocs.publisher.awsS3.region');
+
+    // AWS endpoint is an optional config. If missing, the default endpoint is built from
+    // the configured region.
+    const endpoint = config.getOptionalString(
+      'techdocs.publisher.awsS3.endpoint',
+    );
+
+    // AWS forcePathStyle is an optional config. If missing, it defaults to false. Needs to be enabled for cases
+    // where endpoint url points to locally hosted S3 compatible storage like Localstack
+    const s3ForcePathStyle = config.getOptionalBoolean(
+      'techdocs.publisher.awsS3.s3ForcePathStyle',
+    );
+
+    const storageClient = new aws.S3({
+      credentials,
+      ...(region && { region }),
+      ...(endpoint && { endpoint }),
+      ...(s3ForcePathStyle && { s3ForcePathStyle }),
+    });
+
+    const legacyPathCasing =
+      config.getOptionalBoolean(
+        'techdocs.legacyUseCaseSensitiveTripletPaths',
+      ) || false;
+
+    return new AwsS3Publish({
+      storageClient,
+      bucketName,
+      bucketRootPath,
+      legacyPathCasing,
+      logger,
+      sse,
+    });
+  }
+
+  private static buildCredentials(
+    config?: Config,
+  ): Credentials | CredentialsOptions | undefined {
+    if (!config) {
+      return undefined;
+    }
+
+    const accessKeyId = config.getOptionalString('accessKeyId');
+    const secretAccessKey = config.getOptionalString('secretAccessKey');
+    let explicitCredentials: Credentials | undefined;
+    if (accessKeyId && secretAccessKey) {
+      explicitCredentials = new Credentials({
+        accessKeyId,
+        secretAccessKey,
+      });
+    }
+
+    const roleArn = config.getOptionalString('roleArn');
+    if (roleArn) {
+      return new aws.ChainableTemporaryCredentials({
+        masterCredentials: explicitCredentials,
+        params: {
+          RoleSessionName: 'backstage-aws-techdocs-s3-publisher',
+          RoleArn: roleArn,
+        },
+      });
+    }
+
+    return explicitCredentials;
+  }
+
+  /**
+   * Check if the defined bucket exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  async getReadiness(): Promise<ReadinessResponse> {
+    try {
+      await this.storageClient
+        .headBucket({ Bucket: this.bucketName })
+        .promise();
+
+      this.logger.info(
+        `Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
+      );
+
+      return { isAvailable: true };
+    } catch (error) {
+      this.logger.error(
+        `Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` +
+          'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
+          'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
+          'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+      );
+      this.logger.error(`from AWS client library`, error);
+      return {
+        isAvailable: false,
+      };
+    }
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the S3 bucket.
+   * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
+    const useLegacyPathCasing = this.legacyPathCasing;
+    const bucketRootPath = this.bucketRootPath;
+    const sse = this.sse;
+
+    // First, try to retrieve a list of all individual files currently existing
+    let existingFiles: string[] = [];
+    try {
+      const remoteFolder = getCloudPathForLocalPath(
+        entity,
+        undefined,
+        useLegacyPathCasing,
+        bucketRootPath,
+      );
+      existingFiles = await this.getAllObjectsFromBucket({
+        prefix: remoteFolder,
+      });
+    } catch (e) {
+      assertError(e);
+      this.logger.error(
+        `Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
+      );
+    }
+
+    // Then, merge new files into the same folder
+    let absoluteFilesToUpload;
+    try {
+      // Remove the absolute path prefix of the source directory
+      // Path of all files to upload, relative to the root of the source directory
+      // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+      absoluteFilesToUpload = await getFileTreeRecursively(directory);
+
+      await bulkStorageOperation(
+        async absoluteFilePath => {
+          const relativeFilePath = path.relative(directory, absoluteFilePath);
+          const fileStream = fs.createReadStream(absoluteFilePath);
+
+          const params = {
+            Bucket: this.bucketName,
+            Key: getCloudPathForLocalPath(
+              entity,
+              relativeFilePath,
+              useLegacyPathCasing,
+              bucketRootPath,
+            ),
+            Body: fileStream,
+            ...(sse && { ServerSideEncryption: sse }),
+          } as aws.S3.PutObjectRequest;
+
+          objects.push(params.Key);
+          return this.storageClient.upload(params).promise();
+        },
+        absoluteFilesToUpload,
+        { concurrencyLimit: 10 },
+      );
+
+      this.logger.info(
+        `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
+      );
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to AWS S3. ${e}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    // Last, try to remove the files that were *only* present previously
+    try {
+      const relativeFilesToUpload = absoluteFilesToUpload.map(
+        absoluteFilePath =>
+          getCloudPathForLocalPath(
+            entity,
+            path.relative(directory, absoluteFilePath),
+            useLegacyPathCasing,
+            bucketRootPath,
+          ),
+      );
+      const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
+
+      await bulkStorageOperation(
+        async relativeFilePath => {
+          return await this.storageClient
+            .deleteObject({
+              Bucket: this.bucketName,
+              Key: relativeFilePath,
+            })
+            .promise();
+        },
+        staleFiles,
+        { concurrencyLimit: 10 },
+      );
+
+      this.logger.info(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
+      );
+    } catch (error) {
+      const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
+      this.logger.error(errorMessage);
+    }
+    return { objects };
+  }
+
+  async fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata> {
+    try {
+      return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
+        const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+        const entityDir = this.legacyPathCasing
+          ? entityTriplet
+          : lowerCaseEntityTriplet(entityTriplet);
+
+        const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
+
+        const stream = this.storageClient
+          .getObject({
+            Bucket: this.bucketName,
+            Key: `${entityRootDir}/techdocs_metadata.json`,
+          })
+          .createReadStream();
+
+        try {
+          const techdocsMetadataJson = await streamToBuffer(stream);
+          if (!techdocsMetadataJson) {
+            throw new Error(
+              `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
+            );
+          }
+
+          const techdocsMetadata = JSON5.parse(
+            techdocsMetadataJson.toString('utf-8'),
+          );
+
+          resolve(techdocsMetadata);
+        } catch (err) {
+          assertError(err);
+          this.logger.error(err.message);
+          reject(new Error(err.message));
+        }
+      });
+    } catch (e) {
+      throw new ForwardedError('TechDocs metadata fetch failed', e);
+    }
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return async (req, res) => {
+      // Decode and trim the leading forward slash
+      const decodedUri = decodeURI(req.path.replace(/^\//, ''));
+
+      // Root path is removed from the Uri so that legacy casing can be applied
+      // to the entity triplet without manipulating the root path
+      const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri);
+
+      // filePath example - /default/component/documented-component/index.html
+      const filePathNoRoot = this.legacyPathCasing
+        ? decodedUriNoRoot
+        : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot);
+
+      // Re-prepend the root path to the relative file path
+      const filePath = path.posix.join(this.bucketRootPath, filePathNoRoot);
+
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = path.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      const stream = this.storageClient
+        .getObject({ Bucket: this.bucketName, Key: filePath })
+        .createReadStream();
+      try {
+        // Inject response headers
+        for (const [headerKey, headerValue] of Object.entries(
+          responseHeaders,
+        )) {
+          res.setHeader(headerKey, headerValue);
+        }
+
+        res.send(await streamToBuffer(stream));
+      } catch (err) {
+        assertError(err);
+        this.logger.warn(
+          `TechDocs S3 router failed to serve static files from bucket ${this.bucketName} at key ${filePath}: ${err.message}`,
+        );
+        res.status(404).send('File Not Found');
+      }
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    try {
+      const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+      const entityDir = this.legacyPathCasing
+        ? entityTriplet
+        : lowerCaseEntityTriplet(entityTriplet);
+
+      const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
+
+      await this.storageClient
+        .headObject({
+          Bucket: this.bucketName,
+          Key: `${entityRootDir}/index.html`,
+        })
+        .promise();
+      return Promise.resolve(true);
+    } catch (e) {
+      return Promise.resolve(false);
+    }
+  }
+
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    // Iterate through every file in the root of the publisher.
+    const allObjects = await this.getAllObjectsFromBucket();
+    const limiter = createLimiter(concurrency);
+    await Promise.all(
+      allObjects.map(f =>
+        limiter(async file => {
+          let newPath;
+          try {
+            newPath = lowerCaseEntityTripletInStoragePath(file);
+          } catch (e) {
+            assertError(e);
+            this.logger.warn(e.message);
+            return;
+          }
+
+          // If all parts are already lowercase, ignore.
+          if (file === newPath) {
+            return;
+          }
+
+          try {
+            this.logger.verbose(`Migrating ${file}`);
+            await this.storageClient
+              .copyObject({
+                Bucket: this.bucketName,
+                CopySource: [this.bucketName, file].join('/'),
+                Key: newPath,
+              })
+              .promise();
+
+            if (removeOriginal) {
+              await this.storageClient
+                .deleteObject({
+                  Bucket: this.bucketName,
+                  Key: file,
+                })
+                .promise();
+            }
+          } catch (e) {
+            assertError(e);
+            this.logger.warn(`Unable to migrate ${file}: ${e.message}`);
+          }
+        }, f),
+      ),
+    );
+  }
+
+  /**
+   * Returns a list of all object keys from the configured bucket.
+   */
+  protected async getAllObjectsFromBucket(
+    { prefix } = { prefix: '' },
+  ): Promise<string[]> {
+    const objects: string[] = [];
+    let nextContinuation: string | undefined;
+    let allObjects: ListObjectsV2Output;
+    // Iterate through every file in the root of the publisher.
+    do {
+      allObjects = await this.storageClient
+        .listObjectsV2({
+          Bucket: this.bucketName,
+          ContinuationToken: nextContinuation,
+          ...(prefix ? { Prefix: prefix } : {}),
+        })
+        .promise();
+      objects.push(
+        ...(allObjects.Contents || []).map(f => f.Key || '').filter(f => !!f),
+      );
+      nextContinuation = allObjects.NextContinuationToken;
+    } while (nextContinuation);
+
+    return objects;
+  }
+}

plugins/techdocs-node/src/stages/publish/azureBlobStorage.test.ts

@@ -0,0 +1,403 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import express from 'express';
+import request from 'supertest';
+import mockFs from 'mock-fs';
+import path from 'path';
+import fs from 'fs-extra';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
+
+// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library
+
+const rootDir = (global as any).rootDir; // Set by setupTests.ts
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.join(rootDir, namespace || DEFAULT_NAMESPACE, kind, name);
+};
+
+const logger = getVoidLogger();
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+const createPublisherFromConfig = ({
+  accountName = 'accountName',
+  containerName = 'containerName',
+  legacyUseCaseSensitiveTripletPaths = false,
+}: {
+  accountName?: string;
+  containerName?: string;
+  legacyUseCaseSensitiveTripletPaths?: boolean;
+} = {}) => {
+  const config = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7007',
+      publisher: {
+        type: 'azureBlobStorage',
+        azureBlobStorage: {
+          credentials: {
+            accountName,
+            accountKey: 'accountKey',
+          },
+          containerName,
+        },
+      },
+      legacyUseCaseSensitiveTripletPaths,
+    },
+  });
+  return AzureBlobStoragePublish.fromConfig(config, logger);
+};
+
+describe('AzureBlobStoragePublish', () => {
+  const entity = {
+    apiVersion: '1',
+    kind: 'Component',
+    metadata: {
+      name: 'backstage',
+      namespace: 'default',
+      annotations: {},
+    },
+  };
+
+  const entityName = {
+    kind: 'Component',
+    name: 'backstage',
+    namespace: 'default',
+  };
+
+  const techdocsMetadata = {
+    site_name: 'backstage',
+    site_description: 'site_content',
+    etag: 'etag',
+    build_timestamp: 612741599,
+  };
+
+  const directory = getEntityRootDir(entity);
+
+  beforeEach(() => {
+    (logger.error as jest.Mock).mockClear();
+  });
+
+  const files = {
+    'index.html': '',
+    '404.html': '',
+    'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
+    assets: {
+      'main.css': '',
+    },
+    html: {
+      'unsafe.html': '<html></html>',
+    },
+    img: {
+      'with spaces.png': 'found it',
+      'unsafe.svg': '<svg></svg>',
+    },
+    'some folder': {
+      'also with spaces.js': 'found it too',
+    },
+  };
+
+  beforeEach(async () => {
+    mockFs({
+      [directory]: files,
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  describe('getReadiness', () => {
+    it('should validate correct config', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const errorPublisher = createPublisherFromConfig({
+        containerName: 'bad_container',
+      });
+
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
+      });
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
+        ),
+      );
+    });
+  });
+
+  describe('publish', () => {
+    it('should publish a directory', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory as well when legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = path.join(
+        rootDir,
+        'wrong',
+        'path',
+        'to',
+        'generatedDirectory',
+      );
+
+      const publisher = createPublisherFromConfig({
+        containerName: 'bad_container',
+      });
+
+      const fails = publisher.publish({
+        entity,
+        directory: wrongPathToGeneratedDirectory,
+      });
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(
+          'Unable to upload file(s) to Azure. Error: Failed to read template directory: ENOENT, no such file or directory',
+        ),
+      });
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(wrongPathToGeneratedDirectory),
+      });
+    });
+
+    it('reports an error when bad account credentials', async () => {
+      const publisher = createPublisherFromConfig({
+        accountName: 'bad_account_credentials',
+      });
+
+      let error;
+      try {
+        await publisher.publish({ entity, directory });
+      } catch (e: any) {
+        error = e;
+      }
+
+      expect(error.message).toContain(`Unable to upload file(s) to Azure`);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `Unable to upload file(s) to Azure. Error: Upload failed for ${path.join(
+            directory,
+            '404.html',
+          )} with status code 500`,
+        ),
+      );
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should check expected file', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should check expected file when legacy case flag is passed', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(
+        await publisher.hasDocsBeenGenerated({
+          kind: 'triplet',
+          metadata: {
+            namespace: 'invalid',
+            name: 'path',
+          },
+        } as Entity),
+      ).toBe(false);
+    });
+  });
+
+  describe('fetchTechDocsMetadata', () => {
+    it('should return tech docs metadata', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata when json encoded with single quotes', async () => {
+      const techdocsMetadataPath = path.join(
+        directory,
+        'techdocs_metadata.json',
+      );
+      const techdocsMetadataContent = files['techdocs_metadata.json'];
+
+      fs.writeFileSync(
+        techdocsMetadataPath,
+        techdocsMetadataContent.replace(/"/g, "'"),
+      );
+
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+
+      fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent);
+    });
+
+    it('should return an error if the techdocs_metadata.json file is not present', async () => {
+      const publisher = createPublisherFromConfig();
+
+      const invalidEntityName = {
+        namespace: 'invalid',
+        kind: 'triplet',
+        name: 'path',
+      };
+
+      const techDocsMetadaFilePath = path.posix.join(
+        ...Object.values(invalidEntityName),
+        'techdocs_metadata.json',
+      );
+
+      const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
+
+      await expect(fails).rejects.toMatchObject({
+        message: `TechDocs metadata fetch failed; caused by Error: The file ${techDocsMetadaFilePath} does not exist!`,
+      });
+    });
+  });
+
+  describe('docsRouter', () => {
+    const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+
+    let app: express.Express;
+
+    beforeEach(async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should pass expected object path to bucket', async () => {
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass text/plain content-type for html', async () => {
+      const htmlResponse = await request(app).get(
+        `/${entityTripletPath}/html/unsafe.html`,
+      );
+      expect(htmlResponse.text).toEqual('<html></html>');
+      expect(htmlResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+
+      const svgResponse = await request(app).get(
+        `/${entityTripletPath}/img/unsafe.svg`,
+      );
+      expect(svgResponse.text).toEqual('<svg></svg>');
+      expect(svgResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    });
+
+    it('should return 404 if file is not found', async () => {
+      const response = await request(app).get(
+        `/${entityTripletPath}/not-found.html`,
+      );
+      expect(response.status).toBe(404);
+
+      expect(Buffer.from(response.text).toString('utf8')).toEqual(
+        'File Not Found',
+      );
+    });
+  });
+});

plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts

@@ -0,0 +1,465 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { DefaultAzureCredential } from '@azure/identity';
+import {
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from '@azure/storage-blob';
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { assertError, ForwardedError } from '@backstage/errors';
+import express from 'express';
+import JSON5 from 'json5';
+import limiterFactory from 'p-limit';
+import { default as path, default as platformPath } from 'path';
+import { Logger } from 'winston';
+import {
+  bulkStorageOperation,
+  getCloudPathForLocalPath,
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTriplet,
+  getStaleFiles,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
+import {
+  PublisherBase,
+  PublishRequest,
+  PublishResponse,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';
+
+// The number of batches that may be ongoing at the same time.
+const BATCH_CONCURRENCY = 3;
+
+export class AzureBlobStoragePublish implements PublisherBase {
+  private readonly storageClient: BlobServiceClient;
+  private readonly containerName: string;
+  private readonly legacyPathCasing: boolean;
+  private readonly logger: Logger;
+
+  constructor(options: {
+    storageClient: BlobServiceClient;
+    containerName: string;
+    legacyPathCasing: boolean;
+    logger: Logger;
+  }) {
+    this.storageClient = options.storageClient;
+    this.containerName = options.containerName;
+    this.legacyPathCasing = options.legacyPathCasing;
+    this.logger = options.logger;
+  }
+
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
+    let containerName = '';
+    try {
+      containerName = config.getString(
+        'techdocs.publisher.azureBlobStorage.containerName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
+          'techdocs.publisher.azureBlobStorage.containerName is required.',
+      );
+    }
+
+    let accountName = '';
+    try {
+      accountName = config.getString(
+        'techdocs.publisher.azureBlobStorage.credentials.accountName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " +
+          'techdocs.publisher.azureBlobStorage.credentials.accountName is required.',
+      );
+    }
+
+    // Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used.
+    // https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app
+    const accountKey = config.getOptionalString(
+      'techdocs.publisher.azureBlobStorage.credentials.accountKey',
+    );
+
+    let credential;
+    if (accountKey) {
+      credential = new StorageSharedKeyCredential(accountName, accountKey);
+    } else {
+      credential = new DefaultAzureCredential();
+    }
+
+    const storageClient = new BlobServiceClient(
+      `https://${accountName}.blob.core.windows.net`,
+      credential,
+    );
+
+    const legacyPathCasing =
+      config.getOptionalBoolean(
+        'techdocs.legacyUseCaseSensitiveTripletPaths',
+      ) || false;
+
+    return new AzureBlobStoragePublish({
+      storageClient: storageClient,
+      containerName: containerName,
+      legacyPathCasing: legacyPathCasing,
+      logger: logger,
+    });
+  }
+
+  async getReadiness(): Promise<ReadinessResponse> {
+    try {
+      const response = await this.storageClient
+        .getContainerClient(this.containerName)
+        .getProperties();
+
+      if (response._response.status === 200) {
+        return {
+          isAvailable: true,
+        };
+      }
+
+      if (response._response.status >= 400) {
+        this.logger.error(
+          `Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
+        );
+      }
+    } catch (e) {
+      assertError(e);
+      this.logger.error(`from Azure Blob Storage client library: ${e.message}`);
+    }
+
+    this.logger.error(
+      `Could not retrieve metadata about the Azure Blob Storage container ${this.containerName}. ` +
+        'Make sure that the Azure project and container exist and the access key is setup correctly ' +
+        'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
+        'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+    );
+
+    return { isAvailable: false };
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the Azure Blob Storage container.
+   * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
+    const useLegacyPathCasing = this.legacyPathCasing;
+
+    // First, try to retrieve a list of all individual files currently existing
+    const remoteFolder = getCloudPathForLocalPath(
+      entity,
+      undefined,
+      useLegacyPathCasing,
+    );
+    let existingFiles: string[] = [];
+    try {
+      existingFiles = await this.getAllBlobsFromContainer({
+        prefix: remoteFolder,
+        maxPageSize: BATCH_CONCURRENCY,
+      });
+    } catch (e) {
+      assertError(e);
+      this.logger.error(
+        `Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
+      );
+    }
+
+    // Then, merge new files into the same folder
+    let absoluteFilesToUpload;
+    let container: ContainerClient;
+    try {
+      // Remove the absolute path prefix of the source directory
+      // Path of all files to upload, relative to the root of the source directory
+      // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+      absoluteFilesToUpload = await getFileTreeRecursively(directory);
+
+      container = this.storageClient.getContainerClient(this.containerName);
+      const failedOperations: Error[] = [];
+      await bulkStorageOperation(
+        async absoluteFilePath => {
+          const relativeFilePath = path.normalize(
+            path.relative(directory, absoluteFilePath),
+          );
+          const remotePath = getCloudPathForLocalPath(
+            entity,
+            relativeFilePath,
+            useLegacyPathCasing,
+          );
+          objects.push(remotePath);
+          const response = await container
+            .getBlockBlobClient(remotePath)
+            .uploadFile(absoluteFilePath);
+
+          if (response._response.status >= 400) {
+            failedOperations.push(
+              new Error(
+                `Upload failed for ${absoluteFilePath} with status code ${response._response.status}`,
+              ),
+            );
+          }
+
+          return response;
+        },
+        absoluteFilesToUpload,
+        { concurrencyLimit: BATCH_CONCURRENCY },
+      );
+
+      if (failedOperations.length > 0) {
+        throw new Error(
+          failedOperations
+            .map(r => r.message)
+            .filter(Boolean)
+            .join(' '),
+        );
+      }
+
+      this.logger.info(
+        `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
+      );
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to Azure. ${e}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    // Last, try to remove the files that were *only* present previously
+    try {
+      const relativeFilesToUpload = absoluteFilesToUpload.map(
+        absoluteFilePath =>
+          getCloudPathForLocalPath(
+            entity,
+            path.relative(directory, absoluteFilePath),
+            useLegacyPathCasing,
+          ),
+      );
+
+      const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
+
+      await bulkStorageOperation(
+        async relativeFilePath => {
+          return await container.deleteBlob(relativeFilePath);
+        },
+        staleFiles,
+        { concurrencyLimit: BATCH_CONCURRENCY },
+      );
+
+      this.logger.info(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
+      );
+    } catch (error) {
+      const errorMessage = `Unable to delete file(s) from Azure. ${error}`;
+      this.logger.error(errorMessage);
+    }
+
+    return { objects };
+  }
+
+  private download(containerName: string, blobPath: string): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+      const fileStreamChunks: Array<any> = [];
+      this.storageClient
+        .getContainerClient(containerName)
+        .getBlockBlobClient(blobPath)
+        .download()
+        .then(res => {
+          const body = res.readableStreamBody;
+          if (!body) {
+            reject(new Error(`Unable to parse the response data`));
+            return;
+          }
+          body
+            .on('error', reject)
+            .on('data', chunk => {
+              fileStreamChunks.push(chunk);
+            })
+            .on('end', () => {
+              resolve(Buffer.concat(fileStreamChunks));
+            });
+        })
+        .catch(reject);
+    });
+  }
+
+  async fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata> {
+    const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+    const entityRootDir = this.legacyPathCasing
+      ? entityTriplet
+      : lowerCaseEntityTriplet(entityTriplet);
+
+    try {
+      const techdocsMetadataJson = await this.download(
+        this.containerName,
+        `${entityRootDir}/techdocs_metadata.json`,
+      );
+      if (!techdocsMetadataJson) {
+        throw new Error(
+          `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
+        );
+      }
+      const techdocsMetadata = JSON5.parse(
+        techdocsMetadataJson.toString('utf-8'),
+      );
+      return techdocsMetadata;
+    } catch (e) {
+      throw new ForwardedError('TechDocs metadata fetch failed', e);
+    }
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return (req, res) => {
+      // Decode and trim the leading forward slash
+      const decodedUri = decodeURI(req.path.replace(/^\//, ''));
+
+      // filePath example - /default/Component/documented-component/index.html
+      const filePath = this.legacyPathCasing
+        ? decodedUri
+        : lowerCaseEntityTripletInStoragePath(decodedUri);
+
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = platformPath.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      this.download(this.containerName, filePath)
+        .then(fileContent => {
+          // Inject response headers
+          for (const [headerKey, headerValue] of Object.entries(
+            responseHeaders,
+          )) {
+            res.setHeader(headerKey, headerValue);
+          }
+          res.send(fileContent);
+        })
+        .catch(e => {
+          this.logger.warn(
+            `TechDocs Azure router failed to serve content from container ${this.containerName} at path ${filePath}: ${e.message}`,
+          );
+          res.status(404).send('File Not Found');
+        });
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+    const entityRootDir = this.legacyPathCasing
+      ? entityTriplet
+      : lowerCaseEntityTriplet(entityTriplet);
+
+    return this.storageClient
+      .getContainerClient(this.containerName)
+      .getBlockBlobClient(`${entityRootDir}/index.html`)
+      .exists();
+  }
+
+  protected async renameBlob(
+    originalName: string,
+    newName: string,
+    removeOriginal = false,
+  ): Promise<void> {
+    const container = this.storageClient.getContainerClient(this.containerName);
+    const blob = container.getBlobClient(newName);
+    const { url } = container.getBlobClient(originalName);
+    const response = await blob.beginCopyFromURL(url);
+    await response.pollUntilDone();
+    if (removeOriginal) {
+      await container.deleteBlob(originalName);
+    }
+  }
+
+  protected async renameBlobToLowerCase(
+    originalPath: string,
+    removeOriginal: boolean,
+  ) {
+    let newPath;
+    try {
+      newPath = lowerCaseEntityTripletInStoragePath(originalPath);
+    } catch (e) {
+      assertError(e);
+      this.logger.warn(e.message);
+      return;
+    }
+
+    if (originalPath === newPath) return;
+    try {
+      this.logger.verbose(`Migrating ${originalPath}`);
+      await this.renameBlob(originalPath, newPath, removeOriginal);
+    } catch (e) {
+      assertError(e);
+      this.logger.warn(`Unable to migrate ${originalPath}: ${e.message}`);
+    }
+  }
+
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    const promises = [];
+    const limiter = limiterFactory(concurrency);
+    const container = this.storageClient.getContainerClient(this.containerName);
+
+    for await (const blob of container.listBlobsFlat()) {
+      promises.push(
+        limiter(
+          this.renameBlobToLowerCase.bind(this),
+          blob.name,
+          removeOriginal,
+        ),
+      );
+    }
+
+    await Promise.all(promises);
+  }
+
+  protected async getAllBlobsFromContainer({
+    prefix,
+    maxPageSize,
+  }: {
+    prefix: string;
+    maxPageSize: number;
+  }): Promise<string[]> {
+    const blobs: string[] = [];
+    const container = this.storageClient.getContainerClient(this.containerName);
+
+    let iterator = container.listBlobsFlat({ prefix }).byPage({ maxPageSize });
+    let response = (await iterator.next()).value;
+
+    do {
+      for (const blob of response?.segment?.blobItems ?? []) {
+        blobs.push(blob.name);
+      }
+      iterator = container
+        .listBlobsFlat({ prefix })
+        .byPage({ continuationToken: response.continuationToken, maxPageSize });
+      response = (await iterator.next()).value;
+    } while (response && response.continuationToken);
+
+    return blobs;
+  }
+}

plugins/techdocs-node/src/stages/publish/googleStorage.test.ts

@@ -0,0 +1,484 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import express from 'express';
+import request from 'supertest';
+import mockFs from 'mock-fs';
+import path from 'path';
+import fs from 'fs-extra';
+import { GoogleGCSPublish } from './googleStorage';
+
+// NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library
+
+const rootDir = (global as any).rootDir; // Set by setupTests.ts
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.join(rootDir, namespace || DEFAULT_NAMESPACE, kind, name);
+};
+
+const logger = getVoidLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+const createPublisherFromConfig = ({
+  bucketName = 'bucketName',
+  bucketRootPath = '/',
+  legacyUseCaseSensitiveTripletPaths = false,
+}: {
+  bucketName?: string;
+  bucketRootPath?: string;
+  legacyUseCaseSensitiveTripletPaths?: boolean;
+} = {}) => {
+  const config = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7007',
+      publisher: {
+        type: 'googleGcs',
+        googleGcs: {
+          credentials: '{}',
+          bucketName,
+          bucketRootPath,
+        },
+      },
+      legacyUseCaseSensitiveTripletPaths,
+    },
+  });
+  return GoogleGCSPublish.fromConfig(config, logger);
+};
+
+describe('GoogleGCSPublish', () => {
+  const entity = {
+    apiVersion: 'version',
+    kind: 'Component',
+    metadata: {
+      name: 'backstage',
+      namespace: 'default',
+      annotations: {},
+    },
+  };
+
+  const entityName = {
+    kind: 'Component',
+    name: 'backstage',
+    namespace: 'default',
+  };
+
+  const techdocsMetadata = {
+    site_name: 'backstage',
+    site_description: 'site_content',
+    etag: 'etag',
+    build_timestamp: 612741599,
+  };
+
+  const directory = getEntityRootDir(entity);
+
+  const files = {
+    'index.html': '',
+    '404.html': '',
+    assets: {
+      'main.css': '',
+    },
+    'techdocs_metadata.json': JSON.stringify(techdocsMetadata),
+    html: {
+      'unsafe.html': '<html></html>',
+    },
+    img: {
+      'with spaces.png': 'found it',
+      'unsafe.svg': '<svg></svg>',
+    },
+    'some folder': {
+      'also with spaces.js': 'found it too',
+    },
+  };
+
+  beforeAll(() => {
+    mockFs({
+      [directory]: files,
+    });
+  });
+
+  afterAll(() => {
+    mockFs.restore();
+  });
+
+  describe('getReadiness', () => {
+    it('should validate correct config', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketName: 'bad_bucket_name',
+      });
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: false,
+      });
+    });
+  });
+
+  describe('publish', () => {
+    it('should publish a directory', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/component/backstage/404.html',
+          `default/component/backstage/index.html`,
+          `default/component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory as well when legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'default/Component/backstage/404.html',
+          `default/Component/backstage/index.html`,
+          `default/Component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory when root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/component/backstage/404.html',
+          `backstage-data/techdocs/default/component/backstage/index.html`,
+          `backstage-data/techdocs/default/component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should publish a directory when root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      expect(await publisher.publish({ entity, directory })).toMatchObject({
+        objects: expect.arrayContaining([
+          'backstage-data/techdocs/default/Component/backstage/404.html',
+          `backstage-data/techdocs/default/Component/backstage/index.html`,
+          `backstage-data/techdocs/default/Component/backstage/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = path.join(
+        rootDir,
+        'wrong',
+        'path',
+        'to',
+        'generatedDirectory',
+      );
+
+      const publisher = createPublisherFromConfig();
+
+      const fails = publisher.publish({
+        entity,
+        directory: wrongPathToGeneratedDirectory,
+      });
+
+      // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
+      // Issue reported https://github.com/tschaub/mock-fs/issues/118
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(
+          `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`,
+        ),
+      });
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(wrongPathToGeneratedDirectory),
+      });
+    });
+
+    it('should delete stale files after upload', async () => {
+      const bucketName = 'delete_stale_files_success';
+      const publisher = createPublisherFromConfig({ bucketName });
+      await publisher.publish({ entity, directory });
+      expect(logger.info).toHaveBeenLastCalledWith(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`,
+      );
+    });
+
+    it('should log error when the stale files deletion fails', async () => {
+      const bucketName = 'delete_stale_files_error';
+      const publisher = createPublisherFromConfig({ bucketName });
+      await publisher.publish({ entity, directory });
+      expect(logger.error).toHaveBeenLastCalledWith(
+        'Unable to delete file(s) from Google Cloud Storage. Error: Message',
+      );
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const publisher = createPublisherFromConfig();
+      expect(
+        await publisher.hasDocsBeenGenerated({
+          kind: 'entity',
+          metadata: {
+            namespace: 'invalid',
+            name: 'triplet',
+          },
+        } as Entity),
+      ).toBe(false);
+    });
+  });
+
+  describe('fetchTechDocsMetadata', () => {
+    it('should return tech docs metadata', async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata even if root path is specified', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata if root path is specified and legacy casing is used', async () => {
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: 'backstage-data/techdocs',
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+    });
+
+    it('should return tech docs metadata when json encoded with single quotes', async () => {
+      const techdocsMetadataPath = path.join(
+        directory,
+        'techdocs_metadata.json',
+      );
+      const techdocsMetadataContent = files['techdocs_metadata.json'];
+
+      fs.writeFileSync(
+        techdocsMetadataPath,
+        techdocsMetadataContent.replace(/"/g, "'"),
+      );
+
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+
+      expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
+        techdocsMetadata,
+      );
+
+      fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent);
+    });
+
+    it('should return an error if the techdocs_metadata.json file is not present', async () => {
+      const publisher = createPublisherFromConfig();
+
+      const invalidEntityName = {
+        namespace: 'invalid',
+        kind: 'triplet',
+        name: 'path',
+      };
+
+      const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringMatching(/The file .* does not exist/i),
+      });
+    });
+  });
+
+  describe('docsRouter', () => {
+    const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+    // const entityTripletPath =
+
+    let app: Express.Application;
+
+    beforeEach(async () => {
+      const publisher = createPublisherFromConfig();
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+    });
+
+    it('should pass expected object path to bucket', async () => {
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket even if the legacy case is enabled', async () => {
+      const publisher = createPublisherFromConfig({
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => {
+      const rootPath = 'backstage-data/techdocs';
+      const publisher = createPublisherFromConfig({
+        bucketRootPath: rootPath,
+        legacyUseCaseSensitiveTripletPaths: true,
+      });
+      await publisher.publish({ entity, directory });
+      app = express().use(publisher.docsRouter());
+
+      const pngResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${rootPath}/${entityTripletPath}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass text/plain content-type for html', async () => {
+      const htmlResponse = await request(app).get(
+        `/${entityTripletPath}/html/unsafe.html`,
+      );
+      expect(htmlResponse.text).toEqual('<html></html>');
+      expect(htmlResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+
+      const svgResponse = await request(app).get(
+        `/${entityTripletPath}/img/unsafe.svg`,
+      );
+      expect(svgResponse.text).toEqual('<svg></svg>');
+      expect(svgResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    });
+
+    it('should return 404 if file is not found', async () => {
+      const response = await request(app).get(
+        `/${entityTripletPath}/not-found.html`,
+      );
+      expect(response.status).toBe(404);
+
+      expect(Buffer.from(response.text).toString('utf8')).toEqual(
+        'File Not Found',
+      );
+    });
+  });
+});

plugins/techdocs-node/src/stages/publish/googleStorage.ts

@@ -0,0 +1,387 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { assertError } from '@backstage/errors';
+import { File, FileExistsResponse, Storage } from '@google-cloud/storage';
+import express from 'express';
+import JSON5 from 'json5';
+import path from 'path';
+import { Readable } from 'stream';
+import { Logger } from 'winston';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTriplet,
+  lowerCaseEntityTripletInStoragePath,
+  bulkStorageOperation,
+  getCloudPathForLocalPath,
+  getStaleFiles,
+  normalizeExternalStorageRootPath,
+} from './helpers';
+import { MigrateWriteStream } from './migrations';
+import {
+  PublisherBase,
+  PublishRequest,
+  PublishResponse,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';
+
+export class GoogleGCSPublish implements PublisherBase {
+  private readonly storageClient: Storage;
+  private readonly bucketName: string;
+  private readonly legacyPathCasing: boolean;
+  private readonly logger: Logger;
+  private readonly bucketRootPath: string;
+
+  constructor(options: {
+    storageClient: Storage;
+    bucketName: string;
+    legacyPathCasing: boolean;
+    logger: Logger;
+    bucketRootPath: string;
+  }) {
+    this.storageClient = options.storageClient;
+    this.bucketName = options.bucketName;
+    this.legacyPathCasing = options.legacyPathCasing;
+    this.logger = options.logger;
+    this.bucketRootPath = options.bucketRootPath;
+  }
+
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
+    let bucketName = '';
+    try {
+      bucketName = config.getString('techdocs.publisher.googleGcs.bucketName');
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'googleGcs' in your app config, " +
+          'techdocs.publisher.googleGcs.bucketName is required.',
+      );
+    }
+
+    const bucketRootPath = normalizeExternalStorageRootPath(
+      config.getOptionalString('techdocs.publisher.googleGcs.bucketRootPath') ||
+        '',
+    );
+
+    // Credentials is an optional config. If missing, default GCS environment variables will be used.
+    // Read more here https://cloud.google.com/docs/authentication/production
+    const credentials = config.getOptionalString(
+      'techdocs.publisher.googleGcs.credentials',
+    );
+    let credentialsJson: any = {};
+    if (credentials) {
+      try {
+        credentialsJson = JSON.parse(credentials);
+      } catch (err) {
+        throw new Error(
+          'Error in parsing techdocs.publisher.googleGcs.credentials config to JSON.',
+        );
+      }
+    }
+
+    const storageClient = new Storage({
+      ...(credentials && {
+        projectId: credentialsJson.project_id,
+        credentials: credentialsJson,
+      }),
+    });
+
+    const legacyPathCasing =
+      config.getOptionalBoolean(
+        'techdocs.legacyUseCaseSensitiveTripletPaths',
+      ) || false;
+
+    return new GoogleGCSPublish({
+      storageClient,
+      bucketName,
+      legacyPathCasing,
+      logger,
+      bucketRootPath,
+    });
+  }
+
+  /**
+   * Check if the defined bucket exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  async getReadiness(): Promise<ReadinessResponse> {
+    try {
+      await this.storageClient.bucket(this.bucketName).getMetadata();
+      this.logger.info(
+        `Successfully connected to the GCS bucket ${this.bucketName}.`,
+      );
+
+      return {
+        isAvailable: true,
+      };
+    } catch (err) {
+      assertError(err);
+      this.logger.error(
+        `Could not retrieve metadata about the GCS bucket ${this.bucketName}. ` +
+          'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
+          'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
+          'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+      );
+      this.logger.error(`from GCS client library: ${err.message}`);
+
+      return { isAvailable: false };
+    }
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the GCS bucket.
+   * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const objects: string[] = [];
+    const useLegacyPathCasing = this.legacyPathCasing;
+    const bucket = this.storageClient.bucket(this.bucketName);
+    const bucketRootPath = this.bucketRootPath;
+
+    // First, try to retrieve a list of all individual files currently existing
+    let existingFiles: string[] = [];
+    try {
+      const remoteFolder = getCloudPathForLocalPath(
+        entity,
+        undefined,
+        useLegacyPathCasing,
+        bucketRootPath,
+      );
+      existingFiles = await this.getFilesForFolder(remoteFolder);
+    } catch (e) {
+      assertError(e);
+      this.logger.error(
+        `Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
+      );
+    }
+
+    // Then, merge new files into the same folder
+    let absoluteFilesToUpload;
+    try {
+      // Remove the absolute path prefix of the source directory
+      // Path of all files to upload, relative to the root of the source directory
+      // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+      absoluteFilesToUpload = await getFileTreeRecursively(directory);
+
+      await bulkStorageOperation(
+        async absoluteFilePath => {
+          const relativeFilePath = path.relative(directory, absoluteFilePath);
+          const destination = getCloudPathForLocalPath(
+            entity,
+            relativeFilePath,
+            useLegacyPathCasing,
+            bucketRootPath,
+          );
+          objects.push(destination);
+          return await bucket.upload(absoluteFilePath, { destination });
+        },
+        absoluteFilesToUpload,
+        { concurrencyLimit: 10 },
+      );
+
+      this.logger.info(
+        `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
+      );
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to Google Cloud Storage. ${e}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    // Last, try to remove the files that were *only* present previously
+    try {
+      const relativeFilesToUpload = absoluteFilesToUpload.map(
+        absoluteFilePath =>
+          getCloudPathForLocalPath(
+            entity,
+            path.relative(directory, absoluteFilePath),
+            useLegacyPathCasing,
+            bucketRootPath,
+          ),
+      );
+      const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
+
+      await bulkStorageOperation(
+        async relativeFilePath => {
+          return await bucket.file(relativeFilePath).delete();
+        },
+        staleFiles,
+        { concurrencyLimit: 10 },
+      );
+
+      this.logger.info(
+        `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
+      );
+    } catch (error) {
+      const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`;
+      this.logger.error(errorMessage);
+    }
+
+    return { objects };
+  }
+
+  fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata> {
+    return new Promise((resolve, reject) => {
+      const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+      const entityDir = this.legacyPathCasing
+        ? entityTriplet
+        : lowerCaseEntityTriplet(entityTriplet);
+
+      const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
+
+      const fileStreamChunks: Array<any> = [];
+      this.storageClient
+        .bucket(this.bucketName)
+        .file(`${entityRootDir}/techdocs_metadata.json`)
+        .createReadStream()
+        .on('error', err => {
+          this.logger.error(err.message);
+          reject(err);
+        })
+        .on('data', chunk => {
+          fileStreamChunks.push(chunk);
+        })
+        .on('end', () => {
+          const techdocsMetadataJson =
+            Buffer.concat(fileStreamChunks).toString('utf-8');
+          resolve(JSON5.parse(techdocsMetadataJson));
+        });
+    });
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return (req, res) => {
+      // Decode and trim the leading forward slash
+      const decodedUri = decodeURI(req.path.replace(/^\//, ''));
+
+      // Root path is removed from the Uri so that legacy casing can be applied
+      // to the entity triplet without manipulating the root path
+      const decodedUriNoRoot = path.relative(this.bucketRootPath, decodedUri);
+
+      const filePathNoRoot = this.legacyPathCasing
+        ? decodedUriNoRoot
+        : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot);
+
+      // Re-prepend the root path to the relative file path
+      const filePath = path.posix.join(this.bucketRootPath, filePathNoRoot);
+
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = path.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      // Pipe file chunks directly from storage to client.
+      this.storageClient
+        .bucket(this.bucketName)
+        .file(filePath)
+        .createReadStream()
+        .on('pipe', () => {
+          res.writeHead(200, responseHeaders);
+        })
+        .on('error', err => {
+          this.logger.warn(
+            `TechDocs Google GCS router failed to serve content from bucket ${this.bucketName} at path ${filePath}: ${err.message}`,
+          );
+          // Send a 404 with a meaningful message if possible.
+          if (!res.headersSent) {
+            res.status(404).send('File Not Found');
+          } else {
+            res.destroy();
+          }
+        })
+        .pipe(res);
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    return new Promise(resolve => {
+      const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+      const entityDir = this.legacyPathCasing
+        ? entityTriplet
+        : lowerCaseEntityTriplet(entityTriplet);
+
+      const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
+
+      this.storageClient
+        .bucket(this.bucketName)
+        .file(`${entityRootDir}/index.html`)
+        .exists()
+        .then((response: FileExistsResponse) => {
+          resolve(response[0]);
+        })
+        .catch(() => {
+          resolve(false);
+        });
+    });
+  }
+
+  migrateDocsCase({ removeOriginal = false, concurrency = 25 }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      // Iterate through every file in the root of the publisher.
+      const allFileMetadata: Readable = this.storageClient
+        .bucket(this.bucketName)
+        .getFilesStream();
+      const migrateFiles = new MigrateWriteStream(
+        this.logger,
+        removeOriginal,
+        concurrency,
+      );
+      migrateFiles.on('finish', resolve).on('error', reject);
+      allFileMetadata.pipe(migrateFiles).on('error', error => {
+        migrateFiles.destroy();
+        reject(error);
+      });
+    });
+  }
+
+  private getFilesForFolder(folder: string): Promise<string[]> {
+    const fileMetadataStream: Readable = this.storageClient
+      .bucket(this.bucketName)
+      .getFilesStream({ prefix: folder });
+
+    return new Promise((resolve, reject) => {
+      const files: string[] = [];
+
+      fileMetadataStream.on('error', error => {
+        // push file to file array
+        reject(error);
+      });
+
+      fileMetadataStream.on('data', (file: File) => {
+        // push file to file array
+        files.push(file.name);
+      });
+
+      fileMetadataStream.on('end', () => {
+        // resolve promise
+        resolve(files);
+      });
+    });
+  }
+}

plugins/techdocs-node/src/stages/publish/helpers.test.ts

@@ -0,0 +1,307 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import mockFs from 'mock-fs';
+import * as os from 'os';
+import * as path from 'path';
+import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import {
+  getStaleFiles,
+  getFileTreeRecursively,
+  getCloudPathForLocalPath,
+  getHeadersForFileExtension,
+  bulkStorageOperation,
+  lowerCaseEntityTriplet,
+  lowerCaseEntityTripletInStoragePath,
+  normalizeExternalStorageRootPath,
+} from './helpers';
+
+describe('getHeadersForFileExtension', () => {
+  const correctMapOfExtensions = [
+    ['.html', 'text/plain; charset=utf-8'],
+    ['.htm', 'text/plain; charset=utf-8'],
+    ['.HTML', 'text/plain; charset=utf-8'],
+    ['.dhtml', 'text/plain; charset=utf-8'],
+    ['.xhtml', 'text/plain; charset=utf-8'],
+    ['.xml', 'text/plain; charset=utf-8'],
+    ['.css', 'text/css; charset=utf-8'],
+    ['.png', 'image/png'],
+    ['.jpg', 'image/jpeg'],
+    ['.jpeg', 'image/jpeg'],
+    ['.svg', 'text/plain; charset=utf-8'],
+    ['.SVG', 'text/plain; charset=utf-8'],
+    ['.json', 'application/json; charset=utf-8'],
+    ['.this-in-not-an-extension', 'text/plain; charset=utf-8'],
+  ];
+
+  test.each(correctMapOfExtensions)(
+    'check content-type for %s extension',
+    (extension, expectedContentType) => {
+      const headers = getHeadersForFileExtension(extension);
+      expect(headers).toHaveProperty('Content-Type');
+      expect(headers['Content-Type'].toLowerCase()).toBe(expectedContentType);
+    },
+  );
+});
+
+describe('getFileTreeRecursively', () => {
+  const root = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+  beforeEach(() => {
+    mockFs({
+      [root]: {
+        file1: '',
+        subDirA: {
+          file2: '',
+          emptyDir1: mockFs.directory(),
+        },
+        emptyDir2: mockFs.directory(),
+      },
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('returns complete file tree of a path', async () => {
+    const fileList = await getFileTreeRecursively(root);
+    expect(fileList.length).toBe(2);
+    expect(fileList).toContain(path.resolve(root, 'file1'));
+    expect(fileList).toContain(path.resolve(root, 'subDirA/file2'));
+  });
+});
+
+describe('lowerCaseEntityTriplet', () => {
+  it('returns lower-cased entity triplet path', () => {
+    const originalPath = 'default/Component/backstage/index.html';
+    const actualPath = lowerCaseEntityTriplet(originalPath);
+    expect(actualPath).toBe('default/component/backstage/index.html');
+  });
+});
+
+describe('lowerCaseEntityTripletInStoragePath', () => {
+  it('does not lowercase beyond the triplet', () => {
+    const originalPath = 'default/Component/backstage/assets/IMAGE.png';
+    const actualPath = lowerCaseEntityTripletInStoragePath(originalPath);
+    expect(actualPath).toBe('default/component/backstage/assets/IMAGE.png');
+  });
+
+  it('throws error when there is no triplet', () => {
+    const originalPath = '/default/component/IMAGE.png';
+    const error = `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`;
+    expect(() =>
+      lowerCaseEntityTripletInStoragePath(originalPath),
+    ).toThrowError(error);
+  });
+});
+
+describe('normalizeExternalStorageRootPath', () => {
+  it('returns an empty string when empty string provided', () => {
+    const originalPath = '';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('');
+  });
+  it('returns an empty string when only separator is provided', () => {
+    const originalPath = '/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('');
+  });
+  it('returns normalized path from path with leading and trailing sep', () => {
+    const originalPath = '/backstage-data/techdocs/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+  it('returns normalized path from path without leading and trailing sep', () => {
+    const originalPath = 'backstage-data/techdocs';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+  it('returns normalized path from path with trailing sep', () => {
+    const originalPath = 'backstage-data/techdocs/';
+    const normalPath = normalizeExternalStorageRootPath(originalPath);
+    expect(normalPath).toBe('backstage-data/techdocs');
+  });
+});
+
+describe('getStaleFiles', () => {
+  const defaultFiles = [
+    'default/Component/backstage/index.html',
+    'default/Component/backstage/techdocs_metadata.json',
+    'default/Component/backstage/assests/javascripts/bundle.7f4f3c92.min.js',
+    'default/Component/backstage/assets/stylesheets/main.fe0cca5b.min.css',
+  ];
+
+  it('should return empty array if there is no stale file', () => {
+    const oldFiles = [...defaultFiles];
+    const newFiles = [...defaultFiles];
+    const staleFiles = getStaleFiles(newFiles, oldFiles);
+    expect(staleFiles).toHaveLength(0);
+  });
+
+  it('should return all stale files when they exists', () => {
+    const oldFiles = [...defaultFiles, 'stale_file.png'];
+    const newFiles = [...defaultFiles];
+    const staleFiles = getStaleFiles(newFiles, oldFiles);
+    expect(staleFiles).toHaveLength(1);
+    expect(staleFiles).toEqual(expect.arrayContaining(['stale_file.png']));
+  });
+});
+
+describe('getCloudPathForLocalPath', () => {
+  const entity: Entity = {
+    apiVersion: 'version',
+    metadata: { namespace: 'custom', name: 'backstage' },
+    kind: 'Component',
+  };
+
+  it('should compose a remote bucket path including entity information', () => {
+    const remoteBucket = getCloudPathForLocalPath(entity);
+    expect(remoteBucket).toBe('custom/component/backstage/');
+  });
+
+  it('should compose a remote filename including entity information', () => {
+    const localPath = 'index.html';
+    const remoteBucket = getCloudPathForLocalPath(entity, localPath);
+    expect(remoteBucket).toBe(`custom/component/backstage/${localPath}`);
+  });
+
+  it('should use the default namespace when it is undefined', () => {
+    const localPath = 'index.html';
+    const {
+      kind,
+      metadata: { name },
+    } = entity;
+    const remoteBucket = getCloudPathForLocalPath(
+      { kind, metadata: { name } } as Entity,
+      localPath,
+    );
+    expect(remoteBucket).toBe(
+      `${DEFAULT_NAMESPACE}/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should preserve case when legacy flag is passed', () => {
+    const remoteBucket = getCloudPathForLocalPath(entity, undefined, true);
+    expect(remoteBucket).toBe('custom/Component/backstage/');
+  });
+
+  it('should throw error when entity is invalid', () => {
+    expect(() => getCloudPathForLocalPath({} as Entity)).toThrow();
+  });
+
+  it('should prepend root directory to destination', () => {
+    const localPath = 'index/html';
+    const rootPath = 'backstage-data/techdocs/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should add trailing seperator to root directory', () => {
+    const localPath = 'index/html';
+    const rootPath = 'backstage-data/techdocs';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should remove leading seperator from root directory', () => {
+    const localPath = 'index/html';
+    const rootPath = '/backstage-data/techdocs/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(
+      `backstage-data/techdocs/custom/component/backstage/${localPath}`,
+    );
+  });
+
+  it('should ignore seperator if root directory is explicitly defined', () => {
+    const localPath = 'index/html';
+    const rootPath = '/';
+    const remoteBucket = getCloudPathForLocalPath(
+      entity,
+      localPath,
+      false,
+      rootPath,
+    );
+    expect(remoteBucket).toBe(`custom/component/backstage/${localPath}`);
+  });
+});
+
+describe('bulkStorageOperation', () => {
+  const length = 26;
+  const args = Array.from({ length });
+  const createConcurrentRequestCounter = (
+    callback: (count: number) => void,
+  ) => {
+    let count = 0;
+    return () =>
+      new Promise(resolve => {
+        callback(++count);
+        setTimeout(() => {
+          count--;
+          resolve(null);
+        }, 100);
+      });
+  };
+
+  it('should take care of rate limit by default', async () => {
+    const operation = createConcurrentRequestCounter((count: number) => {
+      expect(count <= 25).toBeTruthy();
+    });
+    await bulkStorageOperation(operation, args);
+  });
+
+  it('should accept the number of concurrency limit', async () => {
+    const concurrencyLimit = 10;
+    const operation = createConcurrentRequestCounter((count: number) => {
+      expect(count <= concurrencyLimit).toBeTruthy();
+    });
+    await bulkStorageOperation(operation, args, { concurrencyLimit });
+  });
+
+  it('should wait for all promises be resolved', async () => {
+    const callback = jest.fn();
+    const operation = createConcurrentRequestCounter(callback);
+    await bulkStorageOperation(operation, args);
+    expect(callback).toHaveBeenCalledTimes(length);
+  });
+
+  it('should call operation with the correct argument', async () => {
+    const files = ['file1.txt', 'file2.txt'];
+    const fn = jest.fn();
+    await bulkStorageOperation(fn, files);
+    expect(fn).toHaveBeenCalledTimes(2);
+    expect(fn).toHaveBeenNthCalledWith(1, files[0]);
+    expect(fn).toHaveBeenNthCalledWith(2, files[1]);
+  });
+});

plugins/techdocs-node/src/stages/publish/helpers.ts

@@ -0,0 +1,219 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import mime from 'mime-types';
+import path from 'path';
+import createLimiter from 'p-limit';
+import recursiveReadDir from 'recursive-readdir';
+
+/**
+ * Helper to get the expected content-type for a given file extension. Also
+ * takes XSS mitigation into account.
+ */
+const getContentTypeForExtension = (ext: string): string => {
+  const defaultContentType = 'text/plain; charset=utf-8';
+
+  // Prevent sanitization bypass by preventing browsers from directly rendering
+  // the contents of untrusted files.
+  if (ext.match(/htm|xml|svg/i)) {
+    return defaultContentType;
+  }
+
+  return mime.contentType(ext) || defaultContentType;
+};
+
+export type responseHeadersType = {
+  'Content-Type': string;
+};
+
+/**
+ * Some files need special headers to be used correctly by the frontend. This function
+ * generates headers in the response to those file requests.
+ * @param fileExtension - .html, .css, .js, .png etc.
+ */
+export const getHeadersForFileExtension = (
+  fileExtension: string,
+): responseHeadersType => {
+  return {
+    'Content-Type': getContentTypeForExtension(fileExtension),
+  };
+};
+
+/**
+ * Recursively traverse all the sub-directories of a path and return
+ * a list of absolute paths of all the files. e.g. tree command in Unix
+ *
+ * @example
+ *
+ * /User/username/my_dir
+ *     dirA
+ *     |   subDirA
+ *     |   |   file1
+ *     EmptyDir
+ *     dirB
+ *     |   file2
+ *     file3
+ *
+ * getFileListRecursively('/Users/username/myDir')
+ * // returns
+ * [
+ *   '/User/username/my_dir/dirA/subDirA/file1',
+ *   '/User/username/my_dir/dirB/file2',
+ *   '/User/username/my_dir/file3'
+ * ]
+ * @param rootDirPath - Absolute path to the root directory.
+ */
+export const getFileTreeRecursively = async (
+  rootDirPath: string,
+): Promise<string[]> => {
+  // Iterate on all the files in the directory and its sub-directories
+  const fileList = await recursiveReadDir(rootDirPath).catch(error => {
+    throw new Error(`Failed to read template directory: ${error.message}`);
+  });
+  return fileList;
+};
+
+/**
+ * Takes a posix path and returns a lower-cased version of entity's triplet
+ * with the remaining path in posix.
+ *
+ * Path must not include a starting slash.
+ *
+ * @example
+ * lowerCaseEntityTriplet('default/Component/backstage')
+ * // return default/component/backstage
+ */
+export const lowerCaseEntityTriplet = (posixPath: string): string => {
+  const [namespace, kind, name, ...rest] = posixPath.split(path.posix.sep);
+  const lowerNamespace = namespace.toLowerCase();
+  const lowerKind = kind.toLowerCase();
+  const lowerName = name.toLowerCase();
+  return [lowerNamespace, lowerKind, lowerName, ...rest].join(path.posix.sep);
+};
+
+/**
+ * Takes either a win32 or posix path and returns a lower-cased version of entity's triplet
+ * with the remaining path in posix.
+ *
+ * Starting slashes will be trimmed.
+ *
+ * Throws an error if the path does not appear to be an entity triplet.
+ *
+ * @example
+ * lowerCaseEntityTripletInStoragePath('/default/Component/backstage/file.txt')
+ * // return default/component/backstage/file.txt
+ */
+export const lowerCaseEntityTripletInStoragePath = (
+  originalPath: string,
+): string => {
+  let posixPath = originalPath;
+  if (originalPath.includes(path.win32.sep)) {
+    posixPath = originalPath.split(path.win32.sep).join(path.posix.sep);
+  }
+
+  // remove leading slash
+  const parts = posixPath.split(path.posix.sep);
+  if (parts[0] === '') {
+    parts.shift();
+  }
+
+  // check if all parts of the entity exist (name, namespace, kind) plus filename
+  if (parts.length <= 3) {
+    throw new Error(
+      `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`,
+    );
+  }
+
+  return lowerCaseEntityTriplet(parts.join(path.posix.sep));
+};
+
+/**
+ * Take a posix path and return a path without leading and trailing
+ * separators
+ *
+ * @example
+ * normalizeExternalStorageRootPath('/backstage-data/techdocs/')
+ * // return backstage-data/techdocs
+ */
+export const normalizeExternalStorageRootPath = (posixPath: string): string => {
+  // remove leading slash
+  let normalizedPath = posixPath;
+  if (posixPath.startsWith(path.posix.sep)) {
+    normalizedPath = posixPath.slice(1);
+  }
+
+  // remove trailing slash
+  if (normalizedPath.endsWith(path.posix.sep)) {
+    normalizedPath = normalizedPath.slice(0, normalizedPath.length - 1);
+  }
+
+  return normalizedPath;
+};
+
+// Only returns the files that existed previously and are not present anymore.
+export const getStaleFiles = (
+  newFiles: string[],
+  oldFiles: string[],
+): string[] => {
+  const staleFiles = new Set(oldFiles);
+  newFiles.forEach(newFile => {
+    staleFiles.delete(newFile);
+  });
+  return Array.from(staleFiles);
+};
+
+// Compose actual filename on remote bucket including entity information
+export const getCloudPathForLocalPath = (
+  entity: Entity,
+  localPath = '',
+  useLegacyPathCasing = false,
+  externalStorageRootPath = '',
+): string => {
+  // Convert destination file path to a POSIX path for uploading.
+  // GCS expects / as path separator and relativeFilePath will contain \\ on Windows.
+  // https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork
+  const relativeFilePathPosix = localPath.split(path.sep).join(path.posix.sep);
+
+  // The / delimiter is intentional since it represents the cloud storage and not the local file system.
+  const entityRootDir = `${entity.metadata?.namespace ?? DEFAULT_NAMESPACE}/${
+    entity.kind
+  }/${entity.metadata.name}`;
+
+  const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
+
+  const destination = useLegacyPathCasing
+    ? relativeFilePathTriplet
+    : lowerCaseEntityTriplet(relativeFilePathTriplet);
+
+  // Again, the / delimiter is intentional, as it represents remote storage.
+  const destinationWithRoot = [
+    // The extra filter prevents unintended double slashes and prefixes.
+    ...externalStorageRootPath.split(path.posix.sep).filter(s => s !== ''),
+    destination,
+  ].join('/');
+
+  return destinationWithRoot; // Remote storage file relative path
+};
+
+// Perform rate limited generic operations by passing a function and a list of arguments
+export const bulkStorageOperation = async <T>(
+  operation: (arg: T) => Promise<unknown>,
+  args: T[],
+  { concurrencyLimit } = { concurrencyLimit: 25 },
+) => {
+  const limiter = createLimiter(concurrencyLimit);
+  await Promise.all(args.map(arg => limiter(operation, arg)));
+};

plugins/techdocs-node/src/stages/publish/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { Publisher } from './publish';
+export type {
+  PublisherBase,
+  PublisherType,
+  PublisherFactory,
+  PublishRequest,
+  PublishResponse,
+  MigrateRequest,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';

plugins/techdocs-node/src/stages/publish/local.test.ts

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  getVoidLogger,
+  PluginEndpointDiscovery,
+  resolvePackagePath,
+} from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import express from 'express';
+import request from 'supertest';
+import mockFs from 'mock-fs';
+import * as os from 'os';
+import { LocalPublish } from './local';
+
+const createMockEntity = (annotations = {}, lowerCase = false) => {
+  return {
+    apiVersion: 'version',
+    kind: lowerCase ? 'testkind' : 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const testDiscovery: jest.Mocked<PluginEndpointDiscovery> = {
+  getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007/api/techdocs'),
+  getExternalBaseUrl: jest.fn(),
+};
+
+const logger = getVoidLogger();
+
+const tmpDir =
+  os.platform() === 'win32' ? 'C:\\tmp\\generatedDir' : '/tmp/generatedDir';
+
+const resolvedDir = resolvePackagePath(
+  '@backstage/plugin-techdocs-backend',
+  'static/docs',
+);
+
+describe('local publisher', () => {
+  it('should publish generated documentation dir', async () => {
+    mockFs({
+      [tmpDir]: {
+        'index.html': '',
+      },
+    });
+
+    const mockConfig = new ConfigReader({});
+
+    const publisher = LocalPublish.fromConfig(
+      mockConfig,
+      logger,
+      testDiscovery,
+    );
+    const mockEntity = createMockEntity();
+    const lowerMockEntity = createMockEntity(undefined, true);
+
+    await publisher.publish({ entity: mockEntity, directory: tmpDir });
+
+    expect(await publisher.hasDocsBeenGenerated(mockEntity)).toBe(true);
+
+    // Lower/upper should be treated the same.
+    expect(await publisher.hasDocsBeenGenerated(lowerMockEntity)).toBe(true);
+
+    mockFs.restore();
+  });
+
+  it('should respect legacy casing', async () => {
+    mockFs({
+      [tmpDir]: {
+        'index.html': '',
+      },
+    });
+
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        legacyUseCaseSensitiveTripletPaths: true,
+      },
+    });
+
+    const publisher = LocalPublish.fromConfig(
+      mockConfig,
+      logger,
+      testDiscovery,
+    );
+    const mockEntity = createMockEntity();
+    const lowerMockEntity = createMockEntity(undefined, true);
+
+    await publisher.publish({ entity: mockEntity, directory: tmpDir });
+
+    expect(await publisher.hasDocsBeenGenerated(mockEntity)).toBe(true);
+
+    // Lower/upper should be treated differently.
+    expect(await publisher.hasDocsBeenGenerated(lowerMockEntity)).toBe(false);
+
+    mockFs.restore();
+  });
+
+  describe('docsRouter', () => {
+    const mockConfig = new ConfigReader({});
+    const publisher = LocalPublish.fromConfig(
+      mockConfig,
+      logger,
+      testDiscovery,
+    );
+    let app: express.Express;
+
+    beforeEach(() => {
+      app = express().use(publisher.docsRouter());
+
+      mockFs.restore();
+      mockFs({
+        [resolvedDir]: {
+          'unsafe.html': '<html></html>',
+          'unsafe.svg': '<svg></svg>',
+          default: {
+            testkind: {
+              testname: {
+                'index.html': 'found it',
+              },
+            },
+          },
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should pass text/plain content-type for unsafe types', async () => {
+      const htmlResponse = await request(app).get(`/unsafe.html`);
+      expect(htmlResponse.text).toEqual('<html></html>');
+      expect(htmlResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+
+      const svgResponse = await request(app).get(`/unsafe.svg`);
+      expect(svgResponse.text).toEqual('<svg></svg>');
+      expect(svgResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    });
+
+    it('should redirect case-sensitive triplet path to lower-case', async () => {
+      const response = await request(app)
+        .get('/default/TestKind/TestName/index.html')
+        .expect('Location', '/default/testkind/testname/index.html');
+      expect(response.status).toBe(301);
+    });
+
+    it('should resolve lower-case triplet path content eventually', async () => {
+      const response = await request(app)
+        .get('/default/TestKind/TestName/index.html')
+        .redirects(1);
+      expect(response.text).toEqual('found it');
+    });
+
+    it('should not redirect when legacy case setting is used', async () => {
+      const legacyConfig = new ConfigReader({
+        techdocs: {
+          legacyUseCaseSensitiveTripletPaths: true,
+        },
+      });
+      const legacyPublisher = LocalPublish.fromConfig(
+        legacyConfig,
+        logger,
+        testDiscovery,
+      );
+      app = express().use(legacyPublisher.docsRouter());
+
+      const response = await request(app).get(
+        '/default/TestKind/TestName/index.html',
+      );
+      expect(response.status).toBe(404);
+    });
+  });
+});

plugins/techdocs-node/src/stages/publish/local.ts

@@ -0,0 +1,294 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  PluginEndpointDiscovery,
+  resolvePackagePath,
+} from '@backstage/backend-common';
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import express from 'express';
+import fs from 'fs-extra';
+import os from 'os';
+import createLimiter from 'p-limit';
+import path from 'path';
+import { Logger } from 'winston';
+import {
+  PublisherBase,
+  PublishRequest,
+  PublishResponse,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
+import { assertError } from '@backstage/errors';
+
+// TODO: Use a more persistent storage than node_modules or /tmp directory.
+// Make it configurable with techdocs.publisher.local.publishDirectory
+let staticDocsDir = '';
+try {
+  staticDocsDir = resolvePackagePath(
+    '@backstage/plugin-techdocs-backend',
+    'static/docs',
+  );
+} catch (err) {
+  // This will most probably never be used.
+  // The try/catch is introduced so that techdocs-cli can import @backstage/techdocs-common
+  // on CI/CD without installing techdocs backend plugin.
+  staticDocsDir = os.tmpdir();
+}
+
+/**
+ * Local publisher which uses the local filesystem to store the generated static files. It uses a directory
+ * called "static" at the root of techdocs-backend plugin.
+ */
+export class LocalPublish implements PublisherBase {
+  private readonly legacyPathCasing: boolean;
+  private readonly logger: Logger;
+  private readonly discovery: PluginEndpointDiscovery;
+
+  // TODO: Move the logic of setting staticDocsDir based on config over to
+  // fromConfig, and set the value as a class parameter.
+  constructor(options: {
+    logger: Logger;
+    discovery: PluginEndpointDiscovery;
+    legacyPathCasing: boolean;
+  }) {
+    this.logger = options.logger;
+    this.discovery = options.discovery;
+    this.legacyPathCasing = options.legacyPathCasing;
+  }
+
+  static fromConfig(
+    config: Config,
+    logger: Logger,
+    discovery: PluginEndpointDiscovery,
+  ): PublisherBase {
+    const legacyPathCasing =
+      config.getOptionalBoolean(
+        'techdocs.legacyUseCaseSensitiveTripletPaths',
+      ) || false;
+
+    return new LocalPublish({
+      logger,
+      discovery,
+      legacyPathCasing,
+    });
+  }
+
+  async getReadiness(): Promise<ReadinessResponse> {
+    return {
+      isAvailable: true,
+    };
+  }
+
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    const entityNamespace = entity.metadata.namespace ?? 'default';
+
+    const publishDir = this.staticEntityPathJoin(
+      entityNamespace,
+      entity.kind,
+      entity.metadata.name,
+    );
+
+    if (!fs.existsSync(publishDir)) {
+      this.logger.info(`Could not find ${publishDir}, creating the directory.`);
+      fs.mkdirSync(publishDir, { recursive: true });
+    }
+
+    try {
+      await fs.copy(directory, publishDir);
+      this.logger.info(`Published site stored at ${publishDir}`);
+    } catch (error) {
+      this.logger.debug(
+        `Failed to copy docs from ${directory} to ${publishDir}`,
+      );
+      throw error;
+    }
+
+    // Generate publish response.
+    const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs');
+    const publishedFilePaths = (await getFileTreeRecursively(publishDir)).map(
+      abs => {
+        return abs.split(`${staticDocsDir}/`)[1];
+      },
+    );
+
+    return {
+      remoteUrl: `${techdocsApiUrl}/static/docs/${encodeURIComponent(
+        entity.metadata.name,
+      )}`,
+      objects: publishedFilePaths,
+    };
+  }
+
+  async fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata> {
+    const metadataPath = this.staticEntityPathJoin(
+      entityName.namespace,
+      entityName.kind,
+      entityName.name,
+      'techdocs_metadata.json',
+    );
+
+    try {
+      return await fs.readJson(metadataPath);
+    } catch (err) {
+      assertError(err);
+      this.logger.error(
+        `Unable to read techdocs_metadata.json at ${metadataPath}. Error: ${err}`,
+      );
+      throw new Error(err.message);
+    }
+  }
+
+  docsRouter(): express.Handler {
+    const router = express.Router();
+
+    // Redirect middleware ensuring that requests to case-sensitive entity
+    // triplet paths are always sent to lower-case versions.
+    router.use((req, res, next) => {
+      // If legacy path casing is on, let the request immediately continue.
+      if (this.legacyPathCasing) {
+        return next();
+      }
+
+      // Generate a lower-case entity triplet path.
+      const [_, namespace, kind, name, ...rest] = req.path.split('/');
+
+      // Ignore non-triplet objects.
+      if (!namespace || !kind || !name) {
+        return next();
+      }
+
+      const newPath = [
+        _,
+        namespace.toLowerCase(),
+        kind.toLowerCase(),
+        name.toLowerCase(),
+        ...rest,
+      ].join('/');
+
+      // If there was no change, then let express.static() handle the request.
+      if (newPath === req.path) {
+        return next();
+      }
+
+      // Otherwise, redirect to the new path.
+      return res.redirect(req.baseUrl + newPath, 301);
+    });
+
+    router.use(
+      express.static(staticDocsDir, {
+        // Handle content-type header the same as all other publishers.
+        setHeaders: (res, filePath) => {
+          const fileExtension = path.extname(filePath);
+          const headers = getHeadersForFileExtension(fileExtension);
+          for (const [header, value] of Object.entries(headers)) {
+            res.setHeader(header, value);
+          }
+        },
+      }),
+    );
+
+    return router;
+  }
+
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    const namespace = entity.metadata.namespace ?? 'default';
+
+    const indexHtmlPath = this.staticEntityPathJoin(
+      namespace,
+      entity.kind,
+      entity.metadata.name,
+      'index.html',
+    );
+
+    // Check if the file exists
+    try {
+      await fs.access(indexHtmlPath, fs.constants.F_OK);
+      return true;
+    } catch (err) {
+      return false;
+    }
+  }
+
+  /**
+   * This code will never run in practice. It is merely here to illustrate how
+   * to implement this method for other storage providers.
+   */
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    // Iterate through every file in the root of the publisher.
+    const files = await getFileTreeRecursively(staticDocsDir);
+    const limit = createLimiter(concurrency);
+
+    await Promise.all(
+      files.map(f =>
+        limit(async file => {
+          const relativeFile = file.replace(`${staticDocsDir}${path.sep}`, '');
+          const newFile = lowerCaseEntityTripletInStoragePath(relativeFile);
+
+          // If all parts are already lowercase, ignore.
+          if (relativeFile === newFile) {
+            return;
+          }
+
+          // Otherwise, copy or move the file.
+          await new Promise<void>(resolve => {
+            const migrate = removeOriginal ? fs.move : fs.copyFile;
+            this.logger.verbose(`Migrating ${relativeFile}`);
+            migrate(file, newFile, err => {
+              if (err) {
+                this.logger.warn(
+                  `Unable to migrate ${relativeFile}: ${err.message}`,
+                );
+              }
+              resolve();
+            });
+          });
+        }, f),
+      ),
+    );
+  }
+
+  /**
+   * Utility wrapper around path.join(), used to control legacy case logic.
+   */
+  protected staticEntityPathJoin(...allParts: string[]): string {
+    if (this.legacyPathCasing) {
+      const [namespace, kind, name, ...parts] = allParts;
+      return path.join(staticDocsDir, namespace, kind, name, ...parts);
+    }
+    const [namespace, kind, name, ...parts] = allParts;
+    return path.join(
+      staticDocsDir,
+      namespace.toLowerCase(),
+      kind.toLowerCase(),
+      name.toLowerCase(),
+      ...parts,
+    );
+  }
+}

plugins/techdocs-node/src/stages/publish/migrations/GoogleMigration.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { assertError } from '@backstage/errors';
+import { File } from '@google-cloud/storage';
+import { Writable } from 'stream';
+import { Logger } from 'winston';
+import { lowerCaseEntityTripletInStoragePath } from '../helpers';
+
+/**
+ * Writable stream to handle object copy/move operations. This implementation
+ * ensures we don't read in files from GCS faster than GCS can copy/move them.
+ */
+export class MigrateWriteStream extends Writable {
+  protected logger: Logger;
+  protected removeOriginal: boolean;
+  protected maxConcurrency: number;
+  protected inFlight = 0;
+
+  constructor(logger: Logger, removeOriginal: boolean, concurrency: number) {
+    super({ objectMode: true });
+    this.logger = logger;
+    this.removeOriginal = removeOriginal;
+    this.maxConcurrency = concurrency;
+  }
+
+  _write(file: File, _encoding: BufferEncoding, next: Function) {
+    let shouldCallNext = true;
+    let newFile;
+    try {
+      newFile = lowerCaseEntityTripletInStoragePath(file.name);
+    } catch (e) {
+      assertError(e);
+      this.logger.warn(e.message);
+      next();
+      return;
+    }
+
+    // If all parts are already lowercase, ignore.
+    if (newFile === file.name) {
+      next();
+      return;
+    }
+
+    // Allow up to n-many files to be migrated at a time.
+    this.inFlight++;
+    if (this.inFlight < this.maxConcurrency) {
+      next();
+      shouldCallNext = false;
+    }
+
+    // Otherwise, copy or move the file.
+    const migrate = this.removeOriginal
+      ? file.move.bind(file)
+      : file.copy.bind(file);
+    this.logger.verbose(`Migrating ${file.name}`);
+    migrate(newFile)
+      .catch(e =>
+        this.logger.warn(`Unable to migrate ${file.name}: ${e.message}`),
+      )
+      .finally(() => {
+        this.inFlight--;
+        if (shouldCallNext) {
+          next();
+        }
+      });
+  }
+}

plugins/techdocs-node/src/stages/publish/migrations/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { MigrateWriteStream } from './GoogleMigration';

plugins/techdocs-node/src/stages/publish/openStackSwift.test.ts

@@ -0,0 +1,392 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import {
+  Entity,
+  CompoundEntityRef,
+  DEFAULT_NAMESPACE,
+} from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import express from 'express';
+import request from 'supertest';
+import mockFs from 'mock-fs';
+import os from 'os';
+import path from 'path';
+import { OpenStackSwiftPublish } from './openStackSwift';
+import { PublisherBase, TechDocsMetadata } from './types';
+
+// NOTE: /packages/techdocs-common/__mocks__ is being used to mock @trendyol-js/openstack-swift-sdk client library
+
+const createMockEntity = (annotations = {}): Entity => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      namespace: 'test-namespace',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const createMockEntityName = (): CompoundEntityRef => ({
+  kind: 'TestKind',
+  name: 'test-component-name',
+  namespace: 'test-namespace',
+});
+
+const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.join(rootDir, namespace || DEFAULT_NAMESPACE, kind, name);
+};
+
+const getPosixEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.posix.join(
+    '/rootDir',
+    namespace || DEFAULT_NAMESPACE,
+    kind,
+    name,
+  );
+};
+
+const logger = getVoidLogger();
+
+let publisher: PublisherBase;
+
+beforeEach(() => {
+  mockFs.restore();
+  const mockConfig = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7007',
+      publisher: {
+        type: 'openStackSwift',
+        openStackSwift: {
+          credentials: {
+            id: 'mockid',
+            secret: 'verystrongsecret',
+          },
+          authUrl: 'mockauthurl',
+          swiftUrl: 'mockSwiftUrl',
+          containerName: 'mock',
+        },
+      },
+    },
+  });
+
+  publisher = OpenStackSwiftPublish.fromConfig(mockConfig, logger);
+});
+
+describe('OpenStackSwiftPublish', () => {
+  describe('getReadiness', () => {
+    it('should validate correct config', async () => {
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7007',
+          publisher: {
+            type: 'openStackSwift',
+            openStackSwift: {
+              credentials: {
+                id: 'mockId',
+                secret: 'mockSecret',
+              },
+              authUrl: 'mockauthurl',
+              swiftUrl: 'mockSwiftUrl',
+              containerName: 'errorBucket',
+            },
+          },
+        },
+      });
+
+      const errorPublisher = OpenStackSwiftPublish.fromConfig(
+        mockConfig,
+        logger,
+      );
+
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
+      });
+    });
+  });
+
+  describe('publish', () => {
+    beforeEach(() => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should publish a directory', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      expect(
+        await publisher.publish({
+          entity,
+          directory: entityRootDir,
+        }),
+      ).toMatchObject({
+        objects: expect.arrayContaining([
+          'test-namespace/TestKind/test-component-name/404.html',
+          `test-namespace/TestKind/test-component-name/index.html`,
+          `test-namespace/TestKind/test-component-name/assets/main.css`,
+        ]),
+      });
+    });
+
+    it('should fail to publish a directory', async () => {
+      const wrongPathToGeneratedDirectory = path.join(
+        rootDir,
+        'wrong',
+        'path',
+        'to',
+        'generatedDirectory',
+      );
+
+      const entity = createMockEntity();
+      await expect(
+        publisher.publish({
+          entity,
+          directory: wrongPathToGeneratedDirectory,
+        }),
+      ).rejects.toThrowError();
+
+      const fails = publisher.publish({
+        entity,
+        directory: wrongPathToGeneratedDirectory,
+      });
+
+      // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
+      // Issue reported https://github.com/tschaub/mock-fs/issues/118
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(
+          `Unable to upload file(s) to OpenStack Swift. Error: Failed to read template directory: ENOENT, no such file or directory`,
+        ),
+      });
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringContaining(wrongPathToGeneratedDirectory),
+      });
+
+      mockFs.restore();
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': 'file-content',
+        },
+      });
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+      mockFs.restore();
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const entity = createMockEntity();
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
+    });
+  });
+
+  describe('fetchTechDocsMetadata', () => {
+    it('should return tech docs metadata', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'techdocs_metadata.json':
+            '{"site_name": "backstage", "site_description": "site_content", "etag": "etag", "build_timestamp": 612741599}',
+        },
+      });
+
+      const expectedMetadata: TechDocsMetadata = {
+        site_name: 'backstage',
+        site_description: 'site_content',
+        etag: 'etag',
+        build_timestamp: 612741599,
+      };
+      expect(
+        await publisher.fetchTechDocsMetadata(entityNameMock),
+      ).toStrictEqual(expectedMetadata);
+      mockFs.restore();
+    });
+
+    it('should return tech docs metadata when json encoded with single quotes', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag', 'build_timestamp': 612741599}`,
+        },
+      });
+
+      const expectedMetadata: TechDocsMetadata = {
+        site_name: 'backstage',
+        site_description: 'site_content',
+        etag: 'etag',
+        build_timestamp: 612741599,
+      };
+      expect(
+        await publisher.fetchTechDocsMetadata(entityNameMock),
+      ).toStrictEqual(expectedMetadata);
+      mockFs.restore();
+    });
+
+    it('should return an error if the techdocs_metadata.json file is not present', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getPosixEntityRootDir(entity);
+
+      const fails = publisher.fetchTechDocsMetadata(entityNameMock);
+
+      await expect(fails).rejects.toMatchObject({
+        message: `TechDocs metadata fetch failed, The file ${path.posix.join(
+          entityRootDir,
+          'techdocs_metadata.json',
+        )} does not exist !`,
+      });
+    });
+  });
+
+  describe('docsRouter', () => {
+    let app: express.Express;
+    const entity = createMockEntity();
+    const entityRootDir = getEntityRootDir(entity);
+
+    beforeEach(() => {
+      app = express().use(publisher.docsRouter());
+
+      mockFs.restore();
+      mockFs({
+        [entityRootDir]: {
+          html: {
+            'unsafe.html': '<html></html>',
+          },
+          img: {
+            'unsafe.svg': '<svg></svg>',
+            'with spaces.png': 'found it',
+          },
+          'some folder': {
+            'also with spaces.js': 'found it too',
+          },
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should pass expected object path to bucket', async () => {
+      const {
+        kind,
+        metadata: { namespace, name },
+      } = entity;
+
+      // Ensures leading slash is trimmed and encoded path is decoded.
+      const pngResponse = await request(app).get(
+        `/${namespace}/${kind}/${name}/img/with%20spaces.png`,
+      );
+      expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual(
+        'found it',
+      );
+      const jsResponse = await request(app).get(
+        `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`,
+      );
+      expect(jsResponse.text).toEqual('found it too');
+    });
+
+    it('should pass text/plain content-type for unsafe types', async () => {
+      const {
+        kind,
+        metadata: { namespace, name },
+      } = entity;
+
+      const htmlResponse = await request(app).get(
+        `/${namespace}/${kind}/${name}/html/unsafe.html`,
+      );
+      expect(htmlResponse.text).toEqual('<html></html>');
+      expect(htmlResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+
+      const svgResponse = await request(app).get(
+        `/${namespace}/${kind}/${name}/img/unsafe.svg`,
+      );
+      expect(svgResponse.text).toEqual('<svg></svg>');
+      expect(svgResponse.header).toMatchObject({
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    });
+
+    it('should return 404 if file is not found', async () => {
+      const {
+        kind,
+        metadata: { namespace, name },
+      } = entity;
+
+      const response = await request(app).get(
+        `/${namespace}/${kind}/${name}/not-found.html`,
+      );
+      expect(response.status).toBe(404);
+
+      expect(Buffer.from(response.text).toString('utf8')).toEqual(
+        'File Not Found',
+      );
+    });
+  });
+});

plugins/techdocs-node/src/stages/publish/openStackSwift.ts

@@ -0,0 +1,366 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import express from 'express';
+import fs from 'fs-extra';
+import JSON5 from 'json5';
+import createLimiter from 'p-limit';
+import path from 'path';
+import { SwiftClient } from '@trendyol-js/openstack-swift-sdk';
+import { NotFound } from '@trendyol-js/openstack-swift-sdk/lib/types';
+import { Stream, Readable } from 'stream';
+import { Logger } from 'winston';
+import {
+  getFileTreeRecursively,
+  getHeadersForFileExtension,
+  lowerCaseEntityTripletInStoragePath,
+} from './helpers';
+import {
+  PublisherBase,
+  PublishRequest,
+  PublishResponse,
+  ReadinessResponse,
+  TechDocsMetadata,
+} from './types';
+import { assertError, ForwardedError } from '@backstage/errors';
+
+const streamToBuffer = (stream: Stream | Readable): Promise<Buffer> => {
+  return new Promise((resolve, reject) => {
+    try {
+      const chunks: any[] = [];
+      stream.on('data', chunk => chunks.push(chunk));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(chunks)));
+    } catch (e) {
+      throw new ForwardedError('Unable to parse the response data', e);
+    }
+  });
+};
+
+const bufferToStream = (buffer: Buffer): Readable => {
+  const stream = new Readable();
+  stream.push(buffer);
+  stream.push(null);
+  return stream;
+};
+
+export class OpenStackSwiftPublish implements PublisherBase {
+  private readonly storageClient: SwiftClient;
+  private readonly containerName: string;
+  private readonly logger: Logger;
+
+  constructor(options: {
+    storageClient: SwiftClient;
+    containerName: string;
+    logger: Logger;
+  }) {
+    this.storageClient = options.storageClient;
+    this.containerName = options.containerName;
+    this.logger = options.logger;
+  }
+
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
+    let containerName = '';
+    try {
+      containerName = config.getString(
+        'techdocs.publisher.openStackSwift.containerName',
+      );
+    } catch (error) {
+      throw new Error(
+        "Since techdocs.publisher.type is set to 'openStackSwift' in your app config, " +
+          'techdocs.publisher.openStackSwift.containerName is required.',
+      );
+    }
+
+    const openStackSwiftConfig = config.getConfig(
+      'techdocs.publisher.openStackSwift',
+    );
+
+    const storageClient = new SwiftClient({
+      authEndpoint: openStackSwiftConfig.getString('authUrl'),
+      swiftEndpoint: openStackSwiftConfig.getString('swiftUrl'),
+      credentialId: openStackSwiftConfig.getString('credentials.id'),
+      secret: openStackSwiftConfig.getString('credentials.secret'),
+    });
+
+    return new OpenStackSwiftPublish({ storageClient, containerName, logger });
+  }
+
+  /*
+   * Check if the defined container exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  async getReadiness(): Promise<ReadinessResponse> {
+    try {
+      const container = await this.storageClient.getContainerMetadata(
+        this.containerName,
+      );
+
+      if (!(container instanceof NotFound)) {
+        this.logger.info(
+          `Successfully connected to the OpenStack Swift container ${this.containerName}.`,
+        );
+        return {
+          isAvailable: true,
+        };
+      }
+      this.logger.error(
+        `Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
+          'Make sure the container exists. Also make sure that authentication is setup either by ' +
+          'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
+          'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+      );
+      return {
+        isAvailable: false,
+      };
+    } catch (err) {
+      assertError(err);
+      this.logger.error(`from OpenStack client library: ${err.message}`);
+      return {
+        isAvailable: false,
+      };
+    }
+  }
+
+  /**
+   * Upload all the files from the generated `directory` to the OpenStack Swift container.
+   * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
+   */
+  async publish({
+    entity,
+    directory,
+  }: PublishRequest): Promise<PublishResponse> {
+    try {
+      const objects: string[] = [];
+
+      // Note: OpenStack Swift manages creation of parent directories if they do not exist.
+      // So collecting path of only the files is good enough.
+      const allFilesToUpload = await getFileTreeRecursively(directory);
+      const limiter = createLimiter(10);
+      const uploadPromises: Array<Promise<unknown>> = [];
+      for (const filePath of allFilesToUpload) {
+        // Remove the absolute path prefix of the source directory
+        // Path of all files to upload, relative to the root of the source directory
+        // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
+        const relativeFilePath = path.relative(directory, filePath);
+        // Convert destination file path to a POSIX path for uploading.
+        // Swift expects / as path separator and relativeFilePath will contain \\ on Windows.
+        // https://docs.openstack.org/python-openstackclient/pike/cli/man/openstack.html
+        const relativeFilePathPosix = relativeFilePath
+          .split(path.sep)
+          .join(path.posix.sep);
+
+        // The / delimiter is intentional since it represents the cloud storage and not the local file system.
+        const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+        const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
+        objects.push(destination);
+
+        // Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
+        const uploadFile = limiter(async () => {
+          const fileBuffer = await fs.readFile(filePath);
+          const stream = bufferToStream(fileBuffer);
+          return this.storageClient.upload(
+            this.containerName,
+            destination,
+            stream,
+          );
+        });
+        uploadPromises.push(uploadFile);
+      }
+      await Promise.all(uploadPromises);
+      this.logger.info(
+        `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
+      );
+      return { objects };
+    } catch (e) {
+      const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+  }
+
+  async fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata> {
+    return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
+      const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
+
+      const downloadResponse = await this.storageClient.download(
+        this.containerName,
+        `${entityRootDir}/techdocs_metadata.json`,
+      );
+
+      if (!(downloadResponse instanceof NotFound)) {
+        const stream = downloadResponse.data;
+        try {
+          const techdocsMetadataJson = await streamToBuffer(stream);
+          if (!techdocsMetadataJson) {
+            throw new Error(
+              `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
+            );
+          }
+
+          const techdocsMetadata = JSON5.parse(
+            techdocsMetadataJson.toString('utf-8'),
+          );
+
+          resolve(techdocsMetadata);
+        } catch (err) {
+          assertError(err);
+          this.logger.error(err.message);
+          reject(new Error(err.message));
+        }
+      } else {
+        reject({
+          message: `TechDocs metadata fetch failed, The file /rootDir/${entityRootDir}/techdocs_metadata.json does not exist !`,
+        });
+      }
+    });
+  }
+
+  /**
+   * Express route middleware to serve static files on a route in techdocs-backend.
+   */
+  docsRouter(): express.Handler {
+    return async (req, res) => {
+      // Decode and trim the leading forward slash
+      // filePath example - /default/Component/documented-component/index.html
+      const filePath = decodeURI(req.path.replace(/^\//, ''));
+
+      // Files with different extensions (CSS, HTML) need to be served with different headers
+      const fileExtension = path.extname(filePath);
+      const responseHeaders = getHeadersForFileExtension(fileExtension);
+
+      const downloadResponse = await this.storageClient.download(
+        this.containerName,
+        filePath,
+      );
+
+      if (!(downloadResponse instanceof NotFound)) {
+        const stream = downloadResponse.data;
+
+        try {
+          // Inject response headers
+          for (const [headerKey, headerValue] of Object.entries(
+            responseHeaders,
+          )) {
+            res.setHeader(headerKey, headerValue);
+          }
+
+          res.send(await streamToBuffer(stream));
+        } catch (err) {
+          assertError(err);
+          this.logger.warn(
+            `TechDocs OpenStack swift router failed to serve content from container ${this.containerName} at path ${filePath}: ${err.message}`,
+          );
+          res.status(404).send('File Not Found');
+        }
+      } else {
+        this.logger.warn(
+          `TechDocs OpenStack swift router failed to serve content from container ${this.containerName} at path ${filePath}: Not found`,
+        );
+        res.status(404).send('File Not Found');
+      }
+    };
+  }
+
+  /**
+   * A helper function which checks if index.html of an Entity's docs site is available. This
+   * can be used to verify if there are any pre-generated docs available to serve.
+   */
+  async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
+    const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
+    try {
+      const fileResponse = await this.storageClient.getMetadata(
+        this.containerName,
+        `${entityRootDir}/index.html`,
+      );
+
+      if (!(fileResponse instanceof NotFound)) {
+        return true;
+      }
+      return false;
+    } catch (err) {
+      assertError(err);
+      this.logger.warn(err.message);
+      return false;
+    }
+  }
+
+  async migrateDocsCase({
+    removeOriginal = false,
+    concurrency = 25,
+  }): Promise<void> {
+    // Iterate through every file in the root of the publisher.
+    const allObjects = await this.getAllObjectsFromContainer();
+    const limiter = createLimiter(concurrency);
+    await Promise.all(
+      allObjects.map(f =>
+        limiter(async file => {
+          let newPath;
+          try {
+            newPath = lowerCaseEntityTripletInStoragePath(file);
+          } catch (e) {
+            assertError(e);
+            this.logger.warn(e.message);
+            return;
+          }
+
+          // If all parts are already lowercase, ignore.
+          if (file === newPath) {
+            return;
+          }
+
+          try {
+            this.logger.verbose(`Migrating ${file} to ${newPath}`);
+            await this.storageClient.copy(
+              this.containerName,
+              file,
+              this.containerName,
+              newPath,
+            );
+            if (removeOriginal) {
+              await this.storageClient.delete(this.containerName, file);
+            }
+          } catch (e) {
+            assertError(e);
+            this.logger.warn(`Unable to migrate ${file}: ${e.message}`);
+          }
+        }, f),
+      ),
+    );
+  }
+
+  /**
+   * Returns a list of all object keys from the configured container.
+   */
+  protected async getAllObjectsFromContainer(
+    { prefix } = { prefix: '' },
+  ): Promise<string[]> {
+    let objects: string[] = [];
+    const OSS_MAX_LIMIT = Math.pow(2, 31) - 1;
+
+    const allObjects = await this.storageClient.list(
+      this.containerName,
+      prefix,
+      OSS_MAX_LIMIT,
+    );
+    objects = allObjects.map((object: any) => object.name);
+
+    return objects;
+  }
+}

plugins/techdocs-node/src/stages/publish/publish.test.ts

@@ -0,0 +1,191 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  getVoidLogger,
+  PluginEndpointDiscovery,
+} from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { Publisher } from './publish';
+import { LocalPublish } from './local';
+import { GoogleGCSPublish } from './googleStorage';
+import { AwsS3Publish } from './awsS3';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
+import { OpenStackSwiftPublish } from './openStackSwift';
+
+const logger = getVoidLogger();
+const discovery: jest.Mocked<PluginEndpointDiscovery> = {
+  getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7007'),
+  getExternalBaseUrl: jest.fn(),
+};
+
+describe('Publisher', () => {
+  beforeEach(() => {
+    jest.resetModules(); // clear the cache
+  });
+
+  it('should create local publisher by default', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(LocalPublish);
+  });
+
+  it('should create local publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'local',
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(LocalPublish);
+  });
+
+  it('should create google gcs publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'googleGcs',
+          googleGcs: {
+            credentials: '{}',
+            bucketName: 'bucketName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(GoogleGCSPublish);
+  });
+
+  it('should create AWS S3 publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'awsS3',
+          awsS3: {
+            credentials: {
+              accessKeyId: 'accessKeyId',
+              secretAccessKey: 'secretAccessKey',
+            },
+            bucketName: 'bucketName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AwsS3Publish);
+  });
+
+  it('should create Azure Blob Storage publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'azureBlobStorage',
+          azureBlobStorage: {
+            credentials: {
+              accountName: 'accountName',
+              accountKey: 'accountKey',
+            },
+            containerName: 'containerName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
+  });
+
+  it('should create Azure Blob Storage publisher from environment variables', async () => {
+    process.env.AZURE_TENANT_ID = 'AZURE_TENANT_ID';
+    process.env.AZURE_CLIENT_ID = 'AZURE_CLIENT_ID';
+    process.env.AZURE_CLIENT_SECRET = 'AZURE_CLIENT_SECRET';
+
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'azureBlobStorage',
+          azureBlobStorage: {
+            credentials: {
+              accountName: 'accountName',
+            },
+            containerName: 'containerName',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(AzureBlobStoragePublish);
+  });
+
+  it('should create Open Stack Swift publisher from config', async () => {
+    const mockConfig = new ConfigReader({
+      techdocs: {
+        requestUrl: 'http://localhost:7007',
+        publisher: {
+          type: 'openStackSwift',
+          openStackSwift: {
+            credentials: {
+              id: 'mockId',
+              secret: 'mockSecret',
+            },
+            authUrl: 'mockauthurl',
+            swiftUrl: 'mockSwiftUrl',
+            containerName: 'mock',
+          },
+        },
+      },
+    });
+
+    const publisher = await Publisher.fromConfig(mockConfig, {
+      logger,
+      discovery,
+    });
+    expect(publisher).toBeInstanceOf(OpenStackSwiftPublish);
+  });
+});

plugins/techdocs-node/src/stages/publish/publish.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import { AwsS3Publish } from './awsS3';
+import { AzureBlobStoragePublish } from './azureBlobStorage';
+import { GoogleGCSPublish } from './googleStorage';
+import { LocalPublish } from './local';
+import { OpenStackSwiftPublish } from './openStackSwift';
+import { PublisherFactory, PublisherBase, PublisherType } from './types';
+
+/**
+ * Factory class to create a TechDocs publisher based on defined publisher type in app config.
+ * Uses `techdocs.publisher.type`.
+ * @public
+ */
+export class Publisher {
+  /**
+   * Returns a instance of TechDocs publisher
+   * @param config - A Backstage configuration
+   * @param options - Options for configuring the publisher factory
+   */
+  static async fromConfig(
+    config: Config,
+    { logger, discovery }: PublisherFactory,
+  ): Promise<PublisherBase> {
+    const publisherType = (config.getOptionalString(
+      'techdocs.publisher.type',
+    ) ?? 'local') as PublisherType;
+
+    switch (publisherType) {
+      case 'googleGcs':
+        logger.info('Creating Google Storage Bucket publisher for TechDocs');
+        return GoogleGCSPublish.fromConfig(config, logger);
+      case 'awsS3':
+        logger.info('Creating AWS S3 Bucket publisher for TechDocs');
+        return AwsS3Publish.fromConfig(config, logger);
+      case 'azureBlobStorage':
+        logger.info(
+          'Creating Azure Blob Storage Container publisher for TechDocs',
+        );
+        return AzureBlobStoragePublish.fromConfig(config, logger);
+      case 'openStackSwift':
+        logger.info(
+          'Creating OpenStack Swift Container publisher for TechDocs',
+        );
+        return OpenStackSwiftPublish.fromConfig(config, logger);
+      case 'local':
+        logger.info('Creating Local publisher for TechDocs');
+        return LocalPublish.fromConfig(config, logger, discovery);
+      default:
+        logger.info('Creating Local publisher for TechDocs');
+        return LocalPublish.fromConfig(config, logger, discovery);
+    }
+  }
+}

plugins/techdocs-node/src/stages/publish/types.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Logger } from 'winston';
+import express from 'express';
+
+/**
+ * Options for building publishers
+ * @public
+ */
+export type PublisherFactory = {
+  logger: Logger;
+  discovery: PluginEndpointDiscovery;
+};
+
+/**
+ * Key for all the different types of TechDocs publishers that are supported.
+ * @public
+ */
+export type PublisherType =
+  | 'local'
+  | 'googleGcs'
+  | 'awsS3'
+  | 'azureBlobStorage'
+  | 'openStackSwift';
+
+/**
+ * Request publish definition
+ * @public
+ */
+export type PublishRequest = {
+  entity: Entity;
+  /* The Path to the directory where the generated files are stored. */
+  directory: string;
+};
+
+/**
+ * Response containing metadata about where files were published and what may
+ * have been published or updated.
+ * @public
+ */
+export type PublishResponse = {
+  /**
+   * The URL which serves files from the local publisher's static directory.
+   */
+  remoteUrl?: string;
+  /**
+   * The list of objects (specifically their paths) that were published.
+   * Objects do not have a preceding slash, and match how one would load the
+   * object over the `/static/docs/*` TechDocs Backend Plugin endpoint.
+   */
+  objects?: string[];
+} | void;
+
+/**
+ * Result for the validation check.
+ * @public
+ */
+export type ReadinessResponse = {
+  /** If true, the publisher is able to interact with the backing storage. */
+  isAvailable: boolean;
+};
+
+/**
+ * Type to hold metadata found in techdocs_metadata.json and associated with each site
+ * @param etag - ETag of the resource used to generate the site. Usually the latest commit sha of the source repository.
+ * @public
+ */
+export type TechDocsMetadata = {
+  site_name: string;
+  site_description: string;
+  etag: string;
+  build_timestamp: number;
+  files?: string[];
+};
+
+/**
+ * TechDocs entity triplet migration request
+ * @public
+ */
+export type MigrateRequest = {
+  /**
+   * Whether or not to remove the source file. Defaults to false (acting like a
+   * copy instead of a move).
+   */
+  removeOriginal?: boolean;
+
+  /**
+   * Maximum number of files/objects to migrate at once. Defaults to 25.
+   */
+  concurrency?: number;
+};
+
+/**
+ * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.)
+ * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs.
+ * It also provides APIs to communicate with the storage service.
+ *
+ * @public
+ */
+export interface PublisherBase {
+  /**
+   * Check if the publisher is ready. This check tries to perform certain checks to see if the
+   * publisher is configured correctly and can be used to publish or read documentations.
+   * The different implementations might e.g. use the provided service credentials to access the
+   * target or check if a folder/bucket is available.
+   */
+  getReadiness(): Promise<ReadinessResponse>;
+
+  /**
+   * Store the generated static files onto a storage service (either local filesystem or external service).
+   *
+   * @param request - Object containing the entity from the service
+   *                  catalog, and the directory that contains the generated static files from TechDocs.
+   */
+  publish(request: PublishRequest): Promise<PublishResponse>;
+
+  /**
+   * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc.
+   * This API uses the techdocs_metadata.json file that co-exists along with the generated docs.
+   */
+  fetchTechDocsMetadata(
+    entityName: CompoundEntityRef,
+  ): Promise<TechDocsMetadata>;
+
+  /**
+   * Route middleware to serve static documentation files for an entity.
+   */
+  docsRouter(): express.Handler;
+
+  /**
+   * Check if the index.html is present for the Entity at the Storage location.
+   */
+  hasDocsBeenGenerated(entityName: Entity): Promise<boolean>;
+
+  /**
+   * Migrates documentation objects with case sensitive entity triplets to
+   * lowercase entity triplets. This was (will be) a change introduced in
+   * `techdocs-cli` version `{0.x.y}` and `techdocs-backend` version `{0.x.y}`.
+   *
+   * Implementation of this method is unnecessary in publishers introduced
+   * after version `{0.x.y}` of `techdocs-common`.
+   */
+  migrateDocsCase?(migrateRequest: MigrateRequest): Promise<void>;
+}

plugins/techdocs-node/src/techdocsTypes.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { IndexableDocument } from '@backstage/search-common';
+
+/**
+ * TechDocs indexable document interface
+ * @public
+ */
+export interface TechDocsDocument extends IndexableDocument {
+  /**
+   * Entity kind
+   */
+  kind: string;
+  /**
+   * Entity metadata namespace
+   */
+  namespace: string;
+  /**
+   * Entity metadata name
+   */
+  name: string;
+  /**
+   * Entity lifecycle
+   */
+  lifecycle: string;
+  /**
+   * Entity owner
+   */
+  owner: string;
+  /**
+   * Entity path
+   */
+  path: string;
+}

plugins/techdocs-node/src/testUtils/StorageFilesMock.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import { IStorageFilesMock } from './types';
+
+const rootDir: string = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+const encoding = 'utf8';
+
+export class StorageFilesMock implements IStorageFilesMock {
+  static rootDir = rootDir;
+
+  private files: Record<string, string>;
+
+  constructor() {
+    this.files = {};
+  }
+
+  public emptyFiles(): void {
+    this.files = {};
+  }
+
+  public fileExists(targetPath: string): boolean {
+    const filePath = path.join(rootDir, targetPath);
+    const posixPath = filePath.split(path.posix.sep).join(path.sep);
+    return this.files[posixPath] !== undefined;
+  }
+
+  public readFile(targetPath: string): Buffer {
+    const filePath = path.join(rootDir, targetPath);
+    return Buffer.from(this.files[filePath] ?? '', encoding);
+  }
+
+  public writeFile(targetPath: string, sourcePath: string): void;
+  public writeFile(targetPath: string, sourceBuffer: Buffer): void;
+  public writeFile(targetPath: string, source: string | Buffer): void {
+    const filePath = path.join(rootDir, targetPath);
+    if (typeof source === 'string') {
+      this.files[filePath] = fs.readFileSync(source).toString(encoding);
+    } else {
+      this.files[filePath] = source.toString(encoding);
+    }
+  }
+}

plugins/techdocs-node/src/testUtils/types.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface IStorageFilesMock {
+  emptyFiles(): void;
+  fileExists(targetPath: string): boolean;
+  readFile(targetPath: string): Buffer;
+  writeFile(targetPath: string, sourcePath: string): void;
+  writeFile(targetPath: string, sourceBuffer: Buffer): void;
+  writeFile(targetPath: string, source: string | Buffer): void;
+}