catalog: support supplying a custom catalog descriptor file parser

.changeset/green-boats-attend.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Support supplying a custom catalog descriptor file parser

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -32,6 +32,7 @@ import {
   CatalogProcessorEntityResult,
   CatalogProcessorErrorResult,
   CatalogProcessorLocationResult,
+  CatalogProcessorParser,
   CatalogProcessorResult,
 } from './processors/types';
 import { LocationReader, ReadLocationResult } from './types';
@@ -41,6 +42,7 @@ const MAX_DEPTH = 10;
 
 type Options = {
   reader: UrlReader;
+  parser: CatalogProcessorParser;
   logger: Logger;
   config: Config;
   processors: CatalogProcessor[];
@@ -137,7 +139,6 @@ export class LocationReaders implements LocationReader {
       if (emitResult.type === 'relation') {
         throw new Error('readLocation may not emit entity relations');
       }
-
       emit(emitResult);
     };
 
@@ -149,6 +150,7 @@ export class LocationReaders implements LocationReader {
               item.location,
               item.optional,
               validatedEmit,
+              this.options.parser,
             )
           ) {
             return;

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts

@@ -25,6 +25,7 @@ import {
   CatalogProcessorErrorResult,
   CatalogProcessorResult,
 } from './types';
+import { defaultEntityDataParser } from './util/parse';
 
 describe('UrlReaderProcessor', () => {
   const mockApiOrigin = 'http://localhost';
@@ -52,7 +53,7 @@ describe('UrlReaderProcessor', () => {
     );
 
     const generated = (await new Promise<CatalogProcessorResult>(emit =>
-      processor.readLocation(spec, false, emit),
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
     )) as CatalogProcessorEntityResult;
 
     expect(generated.type).toBe('entity');
@@ -81,7 +82,7 @@ describe('UrlReaderProcessor', () => {
     );
 
     const generated = (await new Promise<CatalogProcessorResult>(emit =>
-      processor.readLocation(spec, false, emit),
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
     )) as CatalogProcessorErrorResult;
 
     expect(generated.type).toBe('error');

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -18,8 +18,11 @@ import { UrlReader } from '@backstage/backend-common';
 import { LocationSpec } from '@backstage/catalog-model';
 import { Logger } from 'winston';
 import * as result from './results';
-import { CatalogProcessor, CatalogProcessorEmit } from './types';
-import { parseEntityYaml } from './util/parse';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+} from './types';
 
 // TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this
 // can be removed in a bit
@@ -43,6 +46,7 @@ export class UrlReaderProcessor implements CatalogProcessor {
     location: LocationSpec,
     optional: boolean,
     emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
   ): Promise<boolean> {
     if (deprecatedTypes.includes(location.type)) {
       // TODO(Rugvip): Remove this warning a month or two into 2021, and remove support for the deprecated types.
@@ -57,7 +61,7 @@ export class UrlReaderProcessor implements CatalogProcessor {
     try {
       const data = await this.options.reader.read(location.target);
 
-      for (const parseResult of parseEntityYaml(data, location)) {
+      for await (const parseResult of parser({ data, location })) {
         emit(parseResult);
       }
     } catch (error) {

plugins/catalog-backend/src/ingestion/processors/types.ts

@@ -27,12 +27,15 @@ export type CatalogProcessor = {
    * @param location The location to read
    * @param optional Whether a missing target should trigger an error
    * @param emit A sink for items resulting from the read
+   * @param parser A parser, that is able to take the raw catalog descriptor
+   *               data and turn it into the actual result pieces.
    * @returns True if handled by this processor, false otherwise
    */
   readLocation?(
     location: LocationSpec,
     optional: boolean,
     emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
   ): Promise<boolean>;
 
   /**
@@ -100,6 +103,16 @@ export type CatalogProcessor = {
   ): Promise<void>;
 };
 
+/**
+ * A parser, that is able to take the raw catalog descriptor data and turn it
+ * into the actual result pieces. The default implementation performs a YAML
+ * document parsing.
+ */
+export type CatalogProcessorParser = (options: {
+  data: Buffer;
+  location: LocationSpec;
+}) => AsyncIterable<CatalogProcessorResult>;
+
 export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void;
 
 export type CatalogProcessorLocationResult = {

plugins/catalog-backend/src/ingestion/processors/util/parse.ts

@@ -18,7 +18,7 @@ import { Entity, LocationSpec } from '@backstage/catalog-model';
 import lodash from 'lodash';
 import yaml from 'yaml';
 import * as result from '../results';
-import { CatalogProcessorResult } from '../types';
+import { CatalogProcessorParser, CatalogProcessorResult } from '../types';
 
 export function* parseEntityYaml(
   data: Buffer,
@@ -50,3 +50,12 @@ export function* parseEntityYaml(
     }
   }
 }
+
+export const defaultEntityDataParser: CatalogProcessorParser = async function* defaultEntityDataParser({
+  data,
+  location,
+}) {
+  for (const e of parseEntityYaml(data, location)) {
+    yield e;
+  }
+};

plugins/catalog-backend/src/service/CatalogBuilder.test.ts

@@ -20,6 +20,7 @@ import { ConfigReader } from '@backstage/config';
 import Knex from 'knex';
 import yaml from 'yaml';
 import { DatabaseManager } from '../database';
+import { CatalogProcessorParser } from '../ingestion';
 import * as result from '../ingestion/processors/results';
 import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder';
 
@@ -209,4 +210,26 @@ describe('CatalogBuilder', () => {
       }),
     ]);
   });
+
+  it('setEntityDataParser works', async () => {
+    const mockParser: CatalogProcessorParser = jest
+      .fn()
+      .mockImplementation(() => {});
+
+    const builder = new CatalogBuilder(env)
+      .setEntityDataParser(mockParser)
+      .replaceProcessors([
+        {
+          async readLocation(_location, _optional, _emit, parser) {
+            expect(parser).toBe(mockParser);
+            return true;
+          },
+        },
+      ]);
+
+    const { higherOrderOperation } = await builder.build();
+    await higherOrderOperation.addLocation({ type: 'x', target: 'y' });
+
+    expect.assertions(1);
+  });
 });

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -39,6 +39,7 @@ import {
   AnnotateLocationEntityProcessor,
   BuiltinKindsEntityProcessor,
   CatalogProcessor,
+  CatalogProcessorParser,
   CodeOwnersProcessor,
   FileReaderProcessor,
   GithubOrgReaderProcessor,
@@ -60,6 +61,7 @@ import {
   textPlaceholderResolver,
   yamlPlaceholderResolver,
 } from '../ingestion/processors/PlaceholderProcessor';
+import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
 import { LocationAnalyzer } from '../ingestion/types';
 
 export type CatalogEnvironment = {
@@ -96,6 +98,7 @@ export class CatalogBuilder {
   private fieldFormatValidators: Partial<Validators>;
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
+  private parser: CatalogProcessorParser | undefined;
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -105,6 +108,7 @@ export class CatalogBuilder {
     this.fieldFormatValidators = {};
     this.processors = [];
     this.processorsReplace = false;
+    this.parser = undefined;
   }
 
   /**
@@ -197,6 +201,20 @@ export class CatalogBuilder {
     return this;
   }
 
+  /**
+   * Sets up the catalog to use a custom parser for entity data.
+   *
+   * This is the function that gets called immediately after some raw entity
+   * specification data has been read from a remote source, and needs to be
+   * parsed and emitted as structured data.
+   *
+   * @param parser The custom parser
+   */
+  setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {
+    this.parser = parser;
+    return this;
+  }
+
   /**
    * Wires up and returns all of the component parts of the catalog
    */
@@ -211,9 +229,11 @@ export class CatalogBuilder {
     const policy = this.buildEntityPolicy();
     const processors = this.buildProcessors();
     const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config);
+    const parser = this.parser || defaultEntityDataParser;
 
     const locationReader = new LocationReaders({
       ...this.env,
+      parser,
       processors,
       rulesEnforcer,
       policy,