TechDocs: Update mkdocs.yml with repo_url when possible before building docs

.changeset/eleven-lobsters-train.md

@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-techdocs': minor
+'@backstage/plugin-techdocs-backend': minor
+---
+
+- Use techdocs annotation to add repo_url if missing in mkdocs.yml. Having repo_url creates a Edit button on techdocs pages.
+- techdocs-backend: API endpoint `/metadata/mkdocs/*` renamed to `/metadata/techdocs/*`

plugins/techdocs-backend/package.json

@@ -32,7 +32,9 @@
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.1",
     "git-url-parse": "^11.4.0",
+    "js-yaml": "^3.14.0",
     "knex": "^0.21.6",
+    "mock-fs": "^4.13.0",
     "nodegit": "^0.27.0",
     "winston": "^3.2.1"
   },

plugins/techdocs-backend/src/service/helpers.ts

@@ -69,10 +69,13 @@ export class DocsBuilder {
     this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
     const preparedDir = await this.preparer.prepare(this.entity);
 
+    const parsedLocationAnnotation = getLocationForEntity(this.entity);
+
     this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
     const { resultDir } = await this.generator.run({
       directory: preparedDir,
       dockerClient: this.dockerClient,
+      parsedLocationAnnotation,
     });
 
     this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);

plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml

@@ -0,0 +1,2 @@
+site_name: Test site name
+site_description: Test site description

plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml

@@ -0,0 +1,4 @@
+site_name: Test site name
+site_description: Test site description
+
+repo_url: https://github.com/backstage/backstage

plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts

@@ -13,10 +13,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import Stream, { PassThrough } from 'stream';
+import fs from 'fs';
 import os from 'os';
+import { resolve as resolvePath } from 'path';
+import Stream, { PassThrough } from 'stream';
 import Docker from 'dockerode';
-import { runDockerContainer, getGeneratorKey } from './helpers';
+import mockFs from 'mock-fs';
+import * as winston from 'winston';
+import {
+  runDockerContainer,
+  getGeneratorKey,
+  isValidRepoUrlForMkdocs,
+  getRepoUrlFromLocationAnnotation,
+  patchMkdocsYmlPreBuild,
+} from './helpers';
+import { RemoteProtocol } from '../prepare/types';
+import { ParsedLocationAnnotation } from '../../../helpers';
 
 const mockEntity = {
   apiVersion: 'version',
@@ -28,6 +40,14 @@ const mockEntity = {
 
 const mockDocker = new Docker() as jest.Mocked<Docker>;
 
+const mkdocsYml = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs.yml'),
+);
+const mkdocsYmlWithRepoUrl = fs.readFileSync(
+  resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'),
+);
+const mockLogger = winston.createLogger();
+
 describe('helpers', () => {
   describe('getGeneratorKey', () => {
     it('should return techdocs as the only generator key', () => {
@@ -138,4 +158,177 @@ describe('helpers', () => {
       });
     });
   });
+
+  describe('isValidRepoUrlForMkdocs', () => {
+    it('should return true for valid repo_url values for mkdocs', () => {
+      const validRepoUrls = [
+        'https://github.com/org/repo',
+        'https://github.com/backstage/backstage/',
+        'https://github.com/org123/repo1-2-3/',
+        'http://github.com/insecureOrg/insecureRepo',
+        'https://gitlab.com/org/repo',
+        'https://gitlab.com/backstage/backstage/',
+        'https://gitlab.com/org123/repo1-2-3/',
+        'http://gitlab.com/insecureOrg/insecureRepo',
+      ];
+
+      const validRemoteProtocols = ['github', 'gitlab'];
+
+      validRepoUrls.forEach(url => {
+        validRemoteProtocols.forEach(targetType => {
+          expect(
+            isValidRepoUrlForMkdocs(url, targetType as RemoteProtocol),
+          ).toBe(true);
+        });
+      });
+    });
+
+    it('should return false for invalid repo_urls values for mkdocs', () => {
+      const invalidRepoUrls = [
+        'git@github.com:org/repo',
+        'https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend',
+      ];
+
+      invalidRepoUrls.forEach(url => {
+        expect(isValidRepoUrlForMkdocs(url, 'github')).toBe(false);
+      });
+    });
+
+    it('should return false for unsupported remote protocols', () => {
+      const validRepoUrl = 'https://github.com/backstage/backstage';
+
+      const unsupportedRemoteProtocols = ['dir', 'file', 'url'];
+
+      unsupportedRemoteProtocols.forEach(targetType => {
+        expect(
+          isValidRepoUrlForMkdocs(validRepoUrl, targetType as RemoteProtocol),
+        ).toBe(false);
+      });
+    });
+  });
+
+  describe('getRepoUrlFromLocationAnnotation', () => {
+    it('should return undefined for unsupported location type', () => {
+      const parsedLocationAnnotation1: ParsedLocationAnnotation = {
+        type: 'dir',
+        target: '/home/user/workspace/docs-repository',
+      };
+
+      const parsedLocationAnnotation2: ParsedLocationAnnotation = {
+        type: 'file',
+        target: '/home/user/workspace/docs-repository/catalog-info.yaml',
+      };
+
+      const parsedLocationAnnotation3: ParsedLocationAnnotation = {
+        type: 'url',
+        target: 'https://my-website.com/storage/this/docs/repository',
+      };
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
+        undefined,
+      );
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
+        undefined,
+      );
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
+        undefined,
+      );
+    });
+
+    it('should return correct target url for supported hosts', () => {
+      const parsedLocationAnnotation1: ParsedLocationAnnotation = {
+        type: 'github',
+        target: 'https://github.com/backstage/backstage.git',
+      };
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
+        'https://github.com/backstage/backstage',
+      );
+
+      const parsedLocationAnnotation2: ParsedLocationAnnotation = {
+        type: 'github',
+        target: 'https://github.com/org/repo',
+      };
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
+        'https://github.com/org/repo',
+      );
+
+      const parsedLocationAnnotation3: ParsedLocationAnnotation = {
+        type: 'gitlab',
+        target: 'https://gitlab.com/org/repo',
+      };
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
+        'https://gitlab.com/org/repo',
+      );
+
+      const parsedLocationAnnotation4: ParsedLocationAnnotation = {
+        type: 'github',
+        target:
+          'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
+      };
+
+      expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation4)).toBe(
+        'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
+      );
+    });
+  });
+
+  describe('pathMkdocsPreBuild', () => {
+    beforeEach(() => {
+      mockFs({
+        '/mkdocs.yml': mkdocsYml,
+        '/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should add repo_url to mkdocs.yml', () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'github',
+        target: 'https://github.com/backstage/backstage',
+      };
+
+      patchMkdocsYmlPreBuild(
+        '/mkdocs.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+      );
+
+      const updatedMkdocsYml = fs.readFileSync('/mkdocs.yml').toString();
+
+      expect(updatedMkdocsYml).toContain(
+        "repo_url: 'https://github.com/backstage/backstage'",
+      );
+    });
+
+    it('should not override existing repo_url in mkdocs.yml', () => {
+      const parsedLocationAnnotation: ParsedLocationAnnotation = {
+        type: 'github',
+        target: 'https://github.com/neworg/newrepo',
+      };
+
+      patchMkdocsYmlPreBuild(
+        '/mkdocs_with_repo_url.yml',
+        mockLogger,
+        parsedLocationAnnotation,
+      );
+
+      const updatedMkdocsYml = fs
+        .readFileSync('/mkdocs_with_repo_url.yml')
+        .toString();
+
+      expect(updatedMkdocsYml).toContain(
+        "repo_url: 'https://github.com/backstage/backstage'",
+      );
+      expect(updatedMkdocsYml).not.toContain(
+        "repo_url: 'https://github.com/neworg/newrepo'",
+      );
+    });
+  });
 });

plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts

@@ -14,11 +14,16 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import fs from 'fs';
+import { spawn } from 'child_process';
 import { Writable, PassThrough } from 'stream';
 import Docker from 'dockerode';
+import yaml from 'js-yaml';
+import { Logger } from 'winston';
+import { Entity } from '@backstage/catalog-model';
 import { SupportedGeneratorKey } from './types';
-import { spawn } from 'child_process';
+import { ParsedLocationAnnotation } from '../../../helpers';
+import { RemoteProtocol } from '../prepare/types';
 
 // TODO: Implement proper support for more generators.
 export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
@@ -142,3 +147,131 @@ export const runCommand = async ({
     });
   });
 };
+
+/**
+ * Return true if mkdocs can compile docs with provided repo_url
+ *
+ * Valid repo_url examples in mkdocs.yml
+ * - https://github.com/backstage/backstage
+ * - https://gitlab.com/org/repo/
+ * - http://github.com/backstage/backstage
+ * - A http(s) protocol URL to the root of the repository
+ *
+ * Invalid repo_url examples in mkdocs.yml
+ * - https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component
+ * - (anything that is not valid as described above)
+ *
+ * @param {string} repoUrl URL supposed to be used as repo_url in mkdocs.yml
+ * @param {RemoteProtocol} locationType Type of source code host - github, gitlab, dir, url, etc.
+ * @returns {boolean}
+ */
+export const isValidRepoUrlForMkdocs = (
+  repoUrl: string,
+  locationType: RemoteProtocol,
+): boolean => {
+  // Trim trailing slash
+  const cleanRepoUrl = repoUrl.replace(/\/$/, '');
+
+  if (locationType === 'github' || locationType === 'gitlab') {
+    // A valid repoUrl to the root of the repository will be split into 5 strings if split using the / delimiter.
+    // We do not want URLs which have more than that number of forward slashes since they will signify a non-root location
+    // Note: This is not the best possible implementation but will work most of the times.. Feel free to improve or
+    // highlight edge cases.
+    return cleanRepoUrl.split('/').length === 5;
+  }
+
+  return false;
+};
+
+/**
+ * Return a valid URL of the repository used in backstage.io/techdocs-ref annotation.
+ * Return undefined if the `target` is not valid in context of repo_url in mkdocs.yml
+ * Alter URL so that it is a valid repo_url config in mkdocs.yml
+ *
+ * @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
+ * @returns {string | undefined}
+ */
+export const getRepoUrlFromLocationAnnotation = (
+  parsedLocationAnnotation: ParsedLocationAnnotation,
+): string | undefined => {
+  const { type: locationType, target } = parsedLocationAnnotation;
+
+  // Add more options from the RemoteProtocol type of parsedLocationAnnotation.type here
+  // when TechDocs supports more hosts and if mkdocs can generated an Edit URL for them.
+  const supportedHosts = ['github', 'gitlab'];
+
+  if (supportedHosts.includes(locationType)) {
+    // Trim .git or .git/ from the end of repository url
+    return target.replace(/.git\/*$/, '');
+  }
+
+  return undefined;
+};
+
+/**
+ * Update the mkdocs.yml file before TechDocs generator uses it to build docs site.
+ *
+ * List of tasks:
+ * - Add repo_url if it does not exists
+ * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default.
+ * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get
+ * the repository URL.
+ *
+ * This function will not throw an error since this is not critical to the whole TechDocs pipeline.
+ * Instead it will log warnings if there are any errors in reading, parsing or writing YAML.
+ *
+ * @param {string} mkdocsYmlPath Absolute path to mkdocs.yml or equivalent of a docs site
+ * @param {Logger} logger
+ * @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
+ */
+export const patchMkdocsYmlPreBuild = async (
+  mkdocsYmlPath: string,
+  logger: Logger,
+  parsedLocationAnnotation: ParsedLocationAnnotation,
+) => {
+  let mkdocsYmlFileString;
+  try {
+    mkdocsYmlFileString = fs.readFileSync(mkdocsYmlPath, 'utf8');
+  } catch (error) {
+    logger.warn(
+      `Could not read file ${mkdocsYmlPath} before running the generator. ${error.message}`,
+    );
+    return;
+  }
+
+  let mkdocsYml: any;
+  try {
+    mkdocsYml = yaml.safeLoad(mkdocsYmlFileString);
+
+    // mkdocsYml should be an object type after successful parsing.
+    // But based on its type definition, it can also be a string or undefined, which we don't want.
+    if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') {
+      throw new Error('Bad YAML format.');
+    }
+  } catch (error) {
+    logger.warn(
+      `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`,
+    );
+    return;
+  }
+
+  // Add repo_url to mkdocs.yml if it is missing. This will enable the Page edit button generated by MkDocs.
+  if (!('repo_url' in mkdocsYml)) {
+    const repoUrl = getRepoUrlFromLocationAnnotation(parsedLocationAnnotation);
+    if (repoUrl !== undefined) {
+      // mkdocs.yml will not build with invalid repo_url. So, make sure it is valid.
+      if (isValidRepoUrlForMkdocs(repoUrl, parsedLocationAnnotation.type)) {
+        mkdocsYml.repo_url = repoUrl;
+      }
+    }
+  }
+
+  try {
+    fs.writeFileSync(mkdocsYmlPath, yaml.safeDump(mkdocsYml), 'utf8');
+  } catch (error) {
+    logger.warn(
+      `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`,
+    );
+    return;
+  }
+};

plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts

@@ -26,7 +26,11 @@ import {
   GeneratorRunOptions,
   GeneratorRunResult,
 } from './types';
-import { runDockerContainer, runCommand } from './helpers';
+import {
+  runDockerContainer,
+  runCommand,
+  patchMkdocsYmlPreBuild,
+} from './helpers';
 
 type TechdocsGeneratorOptions = {
   // This option enables users to configure if they want to use TechDocs container
@@ -62,6 +66,7 @@ export class TechdocsGenerator implements GeneratorBase {
   public async run({
     directory,
     dockerClient,
+    parsedLocationAnnotation,
   }: GeneratorRunOptions): Promise<GeneratorRunResult> {
     const tmpdirPath = os.tmpdir();
     // Fixes a problem with macOS returning a path that is a symlink
@@ -71,6 +76,15 @@ export class TechdocsGenerator implements GeneratorBase {
     );
     const [log, logStream] = createStream();
 
+    // TODO: In future mkdocs.yml can be mkdocs.yaml. So, use a config variable here to find out
+    // the correct file name.
+    // Do some updates to mkdocs.yml before generating docs e.g. adding repo_url
+    await patchMkdocsYmlPreBuild(
+      path.join(directory, 'mkdocs.yml'),
+      this.logger,
+      parsedLocationAnnotation,
+    );
+
     try {
       switch (this.options.runGeneratorIn) {
         case 'local':

plugins/techdocs-backend/src/techdocs/stages/generate/types.ts

@@ -13,9 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import type { Writable } from 'stream';
+import { Writable } from 'stream';
 import Docker from 'dockerode';
 import { Entity } from '@backstage/catalog-model';
+import { ParsedLocationAnnotation } from '../../../helpers';
 
 /**
  * The returned directory from the generator which is ready
@@ -26,14 +27,18 @@ export type GeneratorRunResult = {
 };
 
 /**
- * The values that the generator will receive. The directory of the
- * uncompiled documentation, with the values from the frontend. A dedicated log stream and a docker
- * client to run any generator on top of your directory.
+ * The values that the generator will receive.
+ *
+ * @param {string} directory The directory of the uncompiled documentation, with the values from the frontend
+ * @param {Docker} dockerClient A docker client to run any generator on top of your directory
+ * @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity
+ * @param {Writable} [logStream] A dedicated log stream
  */
 export type GeneratorRunOptions = {
   directory: string;
-  logStream?: Writable;
   dockerClient: Docker;
+  parsedLocationAnnotation: ParsedLocationAnnotation;
+  logStream?: Writable;
 };
 
 export type GeneratorBase = {