feat(techdocs-common): add custom docker image support

Signed-off-by: Andrew Thauer <athauer@wealthsimple.com>

.changeset/red-impalas-serve.md

@@ -0,0 +1,18 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Adds custom docker image support to the techdocs generator. This change adds a new `techdocs.generator` configuration key and deprecates the existing `techdocs.generators.techdocs` key.
+
+```yaml
+techdocs:
+  # recommended, going forward:
+  generator:
+    runIn: 'docker' # or 'local'
+    # New optional settings
+    dockerImage: my-org/techdocs # use a custom docker image
+    pullImage: false # disable automatic pulling of image (e.g. if custom docker login is required)
+  # legacy (deprecated):
+  generators:
+    techdocs: 'docker' # or 'local'
+```

app-config.yaml

@@ -101,8 +101,10 @@ organization:
 # https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach
 techdocs:
   builder: 'local' # Alternatives - 'external'
-  generators:
-    techdocs: 'docker' # Alternatives - 'local'
+  generator:
+    runIn: 'docker'
+    # dockerImage: my-org/techdocs # use a custom docker image
+    # pullImage: true # or false to disable automatic pulling of image (e.g. if custom docker login is required)
   publisher:
     type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage' or 'openStackSwift'. Read documentation for using alternatives.
 

docs/features/techdocs/configuration.md

@@ -13,14 +13,29 @@ configuration options for TechDocs.
 # File: app-config.yaml
 
 techdocs:
-  # generators.techdocs can have two values: 'docker' or 'local'. This is to determine how to run the generator - whether to
-  # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of).
-  # You want to change this to 'local' if you are running Backstage using your own custom Docker setup and want to avoid running
-  # into Docker in Docker situation. Read more here
-  # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional
+  # techdocs.generator is used to configure how documentation sites are generated using MkDocs.
 
-  generators:
-    techdocs: 'docker'
+  generator:
+    # techdocs.generator.runIn can be either 'docker' or 'local'. This is to determine how to run the generator - whether to
+    # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of).
+    # You want to change this to 'local' if you are running Backstage using your own custom Docker setup and want to avoid running
+    # into Docker in Docker situation. Read more here
+    # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional
+
+    runIn: 'docker'
+
+    # techdocs.generator.dockerImage can be used to control the docker image used during documentation generation. This can be useful
+    # if you want to use MkDocs plugins or other packages that are not included in the default techdocs-container (spotify/techdocs).
+    # NOTE: This setting is only used when techdocs.generator.runIn is set to 'docker'.
+
+    dockerImage: 'spotify/techdocs'
+
+    # techdocs.generator.pullImage can be used to disable pulling the latest docker image by default. This can be useful when you are
+    # using a custom techdocs.generator.dockerImage and you have a custom docker login requirement. For example, you need to login to
+    # AWS ECR to pull the docker image.
+    # NOTE: Disabling this requires the docker image was pulled by other means before running the techdocs generator.
+
+    pullImage: true
 
   # techdocs.builder can be either 'local' or 'external.
   # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage

packages/backend-common/api-report.md

@@ -142,6 +142,7 @@ export class DockerContainerRunner implements ContainerRunner {
     mountDirs,
     workingDir,
     envVars,
+    pullImage,
   }: RunContainerOptions): Promise<void>;
 }
 
@@ -374,6 +375,7 @@ export type RunContainerOptions = {
   mountDirs?: Record<string, string>;
   workingDir?: string;
   envVars?: Record<string, string>;
+  pullImage?: boolean;
 };
 
 // @public

packages/backend-common/src/util/ContainerRunner.ts

@@ -24,6 +24,7 @@ export type RunContainerOptions = {
   mountDirs?: Record<string, string>;
   workingDir?: string;
   envVars?: Record<string, string>;
+  pullImage?: boolean;
 };
 
 export interface ContainerRunner {

packages/backend-common/src/util/DockerContainerRunner.test.ts

@@ -58,6 +58,7 @@ describe('DockerContainerRunner', () => {
   });
 
   afterEach(() => {
+    jest.clearAllMocks();
     mockFs.restore();
   });
 
@@ -86,6 +87,17 @@ describe('DockerContainerRunner', () => {
     expect(mockDocker.run).toHaveBeenCalled();
   });
 
+  it('should not pull the docker container when pullImage is false', async () => {
+    await containerTaskApi.runContainer({
+      imageName,
+      args,
+      pullImage: false,
+    });
+
+    expect(mockDocker.pull).not.toHaveBeenCalled();
+    expect(mockDocker.run).toHaveBeenCalled();
+  });
+
   it('should call the dockerClient run command with the correct arguments passed through', async () => {
     await containerTaskApi.runContainer({
       imageName,

packages/backend-common/src/util/DockerContainerRunner.ts

@@ -38,6 +38,7 @@ export class DockerContainerRunner implements ContainerRunner {
     mountDirs = {},
     workingDir,
     envVars = {},
+    pullImage = true,
   }: RunContainerOptions) {
     // Show a better error message when Docker is unavailable.
     try {
@@ -48,15 +49,17 @@ export class DockerContainerRunner implements ContainerRunner {
       );
     }
 
-    await new Promise<void>((resolve, reject) => {
-      this.dockerClient.pull(imageName, {}, (err, stream) => {
-        if (err) return reject(err);
-        stream.pipe(logStream, { end: false });
-        stream.on('end', () => resolve());
-        stream.on('error', (error: Error) => reject(error));
-        return undefined;
+    if (pullImage) {
+      await new Promise<void>((resolve, reject) => {
+        this.dockerClient.pull(imageName, {}, (err, stream) => {
+          if (err) return reject(err);
+          stream.pipe(logStream, { end: false });
+          stream.on('end', () => resolve());
+          stream.on('error', (error: Error) => reject(error));
+          return undefined;
+        });
       });
-    });
+    }
 
     const userOptions: UserOptions = {};
     if (process.getuid && process.getgid) {

packages/techdocs-common/api-report.md

@@ -226,6 +226,17 @@ export class TechdocsGenerator implements GeneratorBase {
     config: Config;
   });
   // (undocumented)
+  static fromConfig(
+    config: Config,
+    {
+      containerRunner,
+      logger,
+    }: {
+      containerRunner: ContainerRunner;
+      logger: Logger_2;
+    },
+  ): Promise<TechdocsGenerator>;
+  // (undocumented)
   run({
     inputDir,
     outputDir,

packages/techdocs-common/src/stages/generate/techdocs.test.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readGeneratorConfig } from './techdocs';
+
+const mockLogger = {
+  warn: jest.fn(),
+};
+
+describe('readGeneratorConfig', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  const logger = mockLogger as any;
+
+  it('defaults to runIn docker', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {},
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: undefined,
+      pullImage: undefined,
+    });
+  });
+
+  it('should read local config', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'local',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'local',
+    });
+  });
+
+  it('should read docker config', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+    });
+  });
+
+  it('should read custom docker image', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+          dockerImage: 'my-org/techdocs',
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: 'my-org/techdocs',
+    });
+  });
+
+  it('should read config disabling docker pull', () => {
+    const config = new ConfigReader({
+      techdocs: {
+        generator: {
+          runIn: 'docker',
+          dockerImage: 'my-org/techdocs',
+          pullImage: false,
+        },
+      },
+    });
+
+    expect(readGeneratorConfig(config, logger)).toEqual({
+      runIn: 'docker',
+      dockerImage: 'my-org/techdocs',
+      pullImage: false,
+    });
+  });
+
+  describe('with legacy techdocs.generators.techdocs config', () => {
+    it('should read legacy docker option', () => {
+      const config = new ConfigReader({
+        techdocs: {
+          generators: {
+            techdocs: 'docker',
+          },
+        },
+      });
+
+      expect(readGeneratorConfig(config, logger)).toEqual({
+        runIn: 'docker',
+      });
+    });
+
+    it('legacy option should log warning', () => {
+      const config = new ConfigReader({
+        techdocs: {
+          generators: {
+            techdocs: 'local',
+          },
+        },
+      });
+
+      expect(readGeneratorConfig(config, logger)).toEqual({
+        runIn: 'local',
+      });
+      expect(logger.warn).toHaveBeenCalledWith(
+        `The 'techdocs.generators.techdocs' configuration key is deprecated and will be removed in the future. Please use 'techdocs.generator' instead.`,
+      );
+    });
+  });
+});

packages/techdocs-common/src/stages/generate/techdocs.ts

@@ -26,14 +26,14 @@ import {
   storeEtagMetadata,
   validateMkdocsYaml,
 } from './helpers';
-import { GeneratorBase, GeneratorRunOptions } from './types';
+import {
+  GeneratorBase,
+  GeneratorConfig,
+  GeneratorRunInType,
+  GeneratorRunOptions,
+} from './types';
 
-type TechdocsGeneratorOptions = {
-  // This option enables users to configure if they want to use TechDocs container
-  // or generate without the container.
-  // This is used to avoid running into Docker in Docker environment.
-  runGeneratorIn: string;
-};
+const defaultDockerImage = 'spotify/techdocs';
 
 const createStream = (): [string[], PassThrough] => {
   const log = [] as Array<string>;
@@ -50,7 +50,17 @@ const createStream = (): [string[], PassThrough] => {
 export class TechdocsGenerator implements GeneratorBase {
   private readonly logger: Logger;
   private readonly containerRunner: ContainerRunner;
-  private readonly options: TechdocsGeneratorOptions;
+  private readonly options: GeneratorConfig;
+
+  static async fromConfig(
+    config: Config,
+    {
+      containerRunner,
+      logger,
+    }: { containerRunner: ContainerRunner; logger: Logger },
+  ) {
+    return new TechdocsGenerator({ logger, containerRunner, config });
+  }
 
   constructor({
     logger,
@@ -62,10 +72,7 @@ export class TechdocsGenerator implements GeneratorBase {
     config: Config;
   }) {
     this.logger = logger;
-    this.options = {
-      runGeneratorIn:
-        config.getOptionalString('techdocs.generators.techdocs') ?? 'docker',
-    };
+    this.options = readGeneratorConfig(config, logger);
     this.containerRunner = containerRunner;
   }
 
@@ -98,7 +105,7 @@ export class TechdocsGenerator implements GeneratorBase {
     };
 
     try {
-      switch (this.options.runGeneratorIn) {
+      switch (this.options.runIn) {
         case 'local':
           await runCommand({
             command: 'mkdocs',
@@ -114,7 +121,7 @@ export class TechdocsGenerator implements GeneratorBase {
           break;
         case 'docker':
           await this.containerRunner.runContainer({
-            imageName: 'spotify/techdocs',
+            imageName: this.options.dockerImage ?? defaultDockerImage,
             args: ['build', '-d', '/output'],
             logStream,
             mountDirs,
@@ -122,6 +129,7 @@ export class TechdocsGenerator implements GeneratorBase {
             // Set the home directory inside the container as something that applications can
             // write to, otherwise they will just fail trying to write to /
             envVars: { HOME: '/tmp' },
+            pullImage: this.options.pullImage,
           });
           this.logger.info(
             `Successfully generated docs from ${inputDir} into ${outputDir} using techdocs-container`,
@@ -129,7 +137,7 @@ export class TechdocsGenerator implements GeneratorBase {
           break;
         default:
           throw new Error(
-            `Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`,
+            `Invalid config value "${this.options.runIn}" provided in 'techdocs.generators.techdocs'.`,
           );
       }
     } catch (error) {
@@ -163,3 +171,27 @@ export class TechdocsGenerator implements GeneratorBase {
     }
   }
 }
+
+export function readGeneratorConfig(
+  config: Config,
+  logger: Logger,
+): GeneratorConfig {
+  const legacyGeneratorType = config.getOptionalString(
+    'techdocs.generators.techdocs',
+  ) as GeneratorRunInType;
+
+  if (legacyGeneratorType) {
+    logger.warn(
+      `The 'techdocs.generators.techdocs' configuration key is deprecated and will be removed in the future. Please use 'techdocs.generator' instead.`,
+    );
+  }
+
+  return {
+    runIn:
+      legacyGeneratorType ??
+      config.getOptionalString('techdocs.generator.runIn') ??
+      'docker',
+    dockerImage: config.getOptionalString('techdocs.generator.dockerImage'),
+    pullImage: config.getOptionalBoolean('techdocs.generator.pullImage'),
+  };
+}

packages/techdocs-common/src/stages/generate/types.ts

@@ -17,6 +17,18 @@ import { Entity } from '@backstage/catalog-model';
 import { Writable } from 'stream';
 import { ParsedLocationAnnotation } from '../../helpers';
 
+// Determines where the generator will be run
+export type GeneratorRunInType = 'docker' | 'local';
+
+/**
+ * The techdocs generator configurations options.
+ */
+export type GeneratorConfig = {
+  runIn: GeneratorRunInType;
+  dockerImage?: string;
+  pullImage?: boolean;
+};
+
 /**
  * The values that the generator will receive.
  *

plugins/techdocs-backend/config.d.ts

@@ -29,7 +29,31 @@ export interface Config {
     /**
      * Techdocs generator information
      */
+    generator?: {
+      /**
+       * Where to run the techdocs (mkdocs) generator
+       */
+      runIn: 'local' | 'docker';
+
+      /**
+       * Override the default techdocs docker image
+       */
+      dockerImage?: string;
+
+      /**
+       * Pull the latest docker image
+       */
+      pullImage?: boolean;
+    };
+
+    /**
+     * Techdocs generator information
+     * @deprecated Replaced with techdocs.generator
+     */
     generators?: {
+      /**
+       * @deprecated Use techdocs.generator.runIn
+       */
       techdocs: 'local' | 'docker';
     };