diff --git a/.changeset/green-boats-attend.md b/.changeset/green-boats-attend.md new file mode 100644 index 0000000000..ae2324291e --- /dev/null +++ b/.changeset/green-boats-attend.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Support supplying a custom catalog descriptor file parser diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index cb703b8e15..046b6bdf76 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,6 +32,7 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorLocationResult, + CatalogProcessorParser, CatalogProcessorResult, } from './processors/types'; import { LocationReader, ReadLocationResult } from './types'; @@ -41,6 +42,7 @@ const MAX_DEPTH = 10; type Options = { reader: UrlReader; + parser: CatalogProcessorParser; logger: Logger; config: Config; processors: CatalogProcessor[]; @@ -137,7 +139,6 @@ export class LocationReaders implements LocationReader { if (emitResult.type === 'relation') { throw new Error('readLocation may not emit entity relations'); } - emit(emitResult); }; @@ -149,6 +150,7 @@ export class LocationReaders implements LocationReader { item.location, item.optional, validatedEmit, + this.options.parser, ) ) { return; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index aac4235e3a..49b6fe71e0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -25,6 +25,7 @@ import { CatalogProcessorErrorResult, CatalogProcessorResult, } from './types'; +import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; @@ -52,7 +53,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); @@ -81,7 +82,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorErrorResult; expect(generated.type).toBe('error'); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 04f5ee738a..9daae29d3b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -18,8 +18,11 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger } from 'winston'; import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { parseEntityYaml } from './util/parse'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; // TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this // can be removed in a bit @@ -43,6 +46,7 @@ export class UrlReaderProcessor implements CatalogProcessor { location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise { if (deprecatedTypes.includes(location.type)) { // TODO(Rugvip): Remove this warning a month or two into 2021, and remove support for the deprecated types. @@ -57,7 +61,7 @@ export class UrlReaderProcessor implements CatalogProcessor { try { const data = await this.options.reader.read(location.target); - for (const parseResult of parseEntityYaml(data, location)) { + for await (const parseResult of parser({ data, location })) { emit(parseResult); } } catch (error) { diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 1bf30567bc..f7e11d5616 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -27,12 +27,15 @@ export type CatalogProcessor = { * @param location The location to read * @param optional Whether a missing target should trigger an error * @param emit A sink for items resulting from the read + * @param parser A parser, that is able to take the raw catalog descriptor + * data and turn it into the actual result pieces. * @returns True if handled by this processor, false otherwise */ readLocation?( location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise; /** @@ -100,6 +103,16 @@ export type CatalogProcessor = { ): Promise; }; +/** + * A parser, that is able to take the raw catalog descriptor data and turn it + * into the actual result pieces. The default implementation performs a YAML + * document parsing. + */ +export type CatalogProcessorParser = (options: { + data: Buffer; + location: LocationSpec; +}) => AsyncIterable; + export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void; export type CatalogProcessorLocationResult = { diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts index c3dcd42d62..aa24968d6d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts @@ -18,7 +18,7 @@ import { Entity, LocationSpec } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; import * as result from '../results'; -import { CatalogProcessorResult } from '../types'; +import { CatalogProcessorParser, CatalogProcessorResult } from '../types'; export function* parseEntityYaml( data: Buffer, @@ -50,3 +50,12 @@ export function* parseEntityYaml( } } } + +export const defaultEntityDataParser: CatalogProcessorParser = async function* defaultEntityDataParser({ + data, + location, +}) { + for (const e of parseEntityYaml(data, location)) { + yield e; + } +}; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 37337f76f1..695902fec9 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -20,6 +20,7 @@ import { ConfigReader } from '@backstage/config'; import Knex from 'knex'; import yaml from 'yaml'; import { DatabaseManager } from '../database'; +import { CatalogProcessorParser } from '../ingestion'; import * as result from '../ingestion/processors/results'; import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder'; @@ -209,4 +210,26 @@ describe('CatalogBuilder', () => { }), ]); }); + + it('setEntityDataParser works', async () => { + const mockParser: CatalogProcessorParser = jest + .fn() + .mockImplementation(() => {}); + + const builder = new CatalogBuilder(env) + .setEntityDataParser(mockParser) + .replaceProcessors([ + { + async readLocation(_location, _optional, _emit, parser) { + expect(parser).toBe(mockParser); + return true; + }, + }, + ]); + + const { higherOrderOperation } = await builder.build(); + await higherOrderOperation.addLocation({ type: 'x', target: 'y' }); + + expect.assertions(1); + }); }); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f6170135fd..938a406e56 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -39,6 +39,7 @@ import { AnnotateLocationEntityProcessor, BuiltinKindsEntityProcessor, CatalogProcessor, + CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, GithubOrgReaderProcessor, @@ -60,6 +61,7 @@ import { textPlaceholderResolver, yamlPlaceholderResolver, } from '../ingestion/processors/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; export type CatalogEnvironment = { @@ -96,6 +98,7 @@ export class CatalogBuilder { private fieldFormatValidators: Partial; private processors: CatalogProcessor[]; private processorsReplace: boolean; + private parser: CatalogProcessorParser | undefined; constructor(env: CatalogEnvironment) { this.env = env; @@ -105,6 +108,7 @@ export class CatalogBuilder { this.fieldFormatValidators = {}; this.processors = []; this.processorsReplace = false; + this.parser = undefined; } /** @@ -197,6 +201,20 @@ export class CatalogBuilder { return this; } + /** + * Sets up the catalog to use a custom parser for entity data. + * + * This is the function that gets called immediately after some raw entity + * specification data has been read from a remote source, and needs to be + * parsed and emitted as structured data. + * + * @param parser The custom parser + */ + setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder { + this.parser = parser; + return this; + } + /** * Wires up and returns all of the component parts of the catalog */ @@ -211,9 +229,11 @@ export class CatalogBuilder { const policy = this.buildEntityPolicy(); const processors = this.buildProcessors(); const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config); + const parser = this.parser || defaultEntityDataParser; const locationReader = new LocationReaders({ ...this.env, + parser, processors, rulesEnforcer, policy,