diff --git a/.changeset/young-peaches-shake.md b/.changeset/young-peaches-shake.md new file mode 100644 index 0000000000..75c93f8f3b --- /dev/null +++ b/.changeset/young-peaches-shake.md @@ -0,0 +1,5 @@ +--- +'@backstage/config-loader': minor +--- + +Add configuration key to File and Remote `ConfigSource`s that enables configuration of parsing logic. Previously limited to yaml, these `ConfigSource`s now allow for a multitude of parsing options (e.g. JSON). diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index bc8f217bbb..7b7ab5a36d 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -141,6 +141,7 @@ export class FileConfigSource implements ConfigSource { // @public export interface FileConfigSourceOptions { + parser?: Parser; path: string; substitutionFunc?: EnvFunc; watch?: boolean; @@ -218,6 +219,11 @@ export interface MutableConfigSourceOptions { data?: JsonObject; } +// @public +export type Parser = ({ contents }: { contents: string }) => Promise<{ + result?: JsonObject; +}>; + // @public export interface ReadConfigDataOptions { // (undocumented) @@ -242,6 +248,7 @@ export class RemoteConfigSource implements ConfigSource { // @public export interface RemoteConfigSourceOptions { + parser?: Parser; reloadInterval?: HumanDuration; substitutionFunc?: EnvFunc; url: string; diff --git a/packages/config-loader/src/sources/FileConfigSource.test.ts b/packages/config-loader/src/sources/FileConfigSource.test.ts index 61cdeed272..4ed31ccee0 100644 --- a/packages/config-loader/src/sources/FileConfigSource.test.ts +++ b/packages/config-loader/src/sources/FileConfigSource.test.ts @@ -59,6 +59,19 @@ describe('FileConfigSource', () => { ]); }); + it('should read a config file with optional parser', async () => { + const tmp = await tmpFiles({ 'a.json': JSON.stringify({ a: 1 }) }); + + const source = FileConfigSource.create({ + path: tmp.resolve('a.json'), + parser: async ({ contents }) => ({ result: JSON.parse(contents) }), + }); + + await expect(readN(source, 1)).resolves.toEqual([ + [{ data: { a: 1 }, context: 'a.json', path: tmp.resolve('a.json') }], + ]); + }); + it('should watch config files', async () => { const tmp = await tmpFiles({ 'a.yaml': 'a: 1' }); diff --git a/packages/config-loader/src/sources/FileConfigSource.ts b/packages/config-loader/src/sources/FileConfigSource.ts index 74266de227..c0697c1180 100644 --- a/packages/config-loader/src/sources/FileConfigSource.ts +++ b/packages/config-loader/src/sources/FileConfigSource.ts @@ -17,16 +17,17 @@ import chokidar, { FSWatcher } from 'chokidar'; import fs from 'fs-extra'; import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; -import yaml from 'yaml'; import { AsyncConfigSourceGenerator, ConfigSource, ConfigSourceData, SubstitutionFunc, + Parser, ReadConfigDataOptions, } from './types'; import { createConfigTransformer } from './transform'; import { NotFoundError } from '@backstage/errors'; +import { parseYamlContent } from './utils'; /** * Options for {@link FileConfigSource.create}. @@ -48,6 +49,11 @@ export interface FileConfigSourceOptions { * A substitution function to use instead of the default environment substitution. */ substitutionFunc?: SubstitutionFunc; + + /** + * A content parsing function to transform string content to configuration values. + */ + parser?: Parser; } async function readFile(path: string): Promise { @@ -95,11 +101,13 @@ export class FileConfigSource implements ConfigSource { readonly #path: string; readonly #substitutionFunc?: SubstitutionFunc; readonly #watch?: boolean; + readonly #parser: Parser; private constructor(options: FileConfigSourceOptions) { this.#path = options.path; this.#substitutionFunc = options.substitutionFunc; this.#watch = options.watch ?? true; + this.#parser = options.parser ?? parseYamlContent; } // Work is duplicated across each read, in practice that should not @@ -154,12 +162,12 @@ export class FileConfigSource implements ConfigSource { watchedPaths.push(this.#path); } - const content = await readFile(this.#path); - if (content === undefined) { + const contents = await readFile(this.#path); + if (contents === undefined) { throw new NotFoundError(`Config file "${this.#path}" does not exist`); } - const parsed = yaml.parse(content); - if (parsed === null) { + const { result: parsed } = await this.#parser({ contents }); + if (parsed === undefined) { return []; } try { diff --git a/packages/config-loader/src/sources/RemoteConfigSource.test.ts b/packages/config-loader/src/sources/RemoteConfigSource.test.ts index e825bf3434..372dc91823 100644 --- a/packages/config-loader/src/sources/RemoteConfigSource.test.ts +++ b/packages/config-loader/src/sources/RemoteConfigSource.test.ts @@ -59,6 +59,45 @@ app: ]); }); + it('should load and parse config from a remote URL', async () => { + worker.use( + rest.get('http://localhost/config.json', (_req, res, ctx) => + res( + ctx.body( + JSON.stringify({ + app: { + title: 'Example App', + substituted: 'x', + escaped: '$${VALUE}', + }, + }), + ), + ), + ), + ); + + const source = RemoteConfigSource.create({ + url: 'http://localhost/config.json', + substitutionFunc: async () => 'x', + parser: async ({ contents }) => ({ result: JSON.parse(contents) }), + }); + + await expect(readN(source, 1)).resolves.toEqual([ + [ + { + context: 'http://localhost/config.json', + data: { + app: { + title: 'Example App', + substituted: 'x', + escaped: '${VALUE}', + }, + }, + }, + ], + ]); + }); + it('should reload config from a remote URL', async () => { let fetched = false; diff --git a/packages/config-loader/src/sources/RemoteConfigSource.ts b/packages/config-loader/src/sources/RemoteConfigSource.ts index 10a64ee635..1492cd4d28 100644 --- a/packages/config-loader/src/sources/RemoteConfigSource.ts +++ b/packages/config-loader/src/sources/RemoteConfigSource.ts @@ -22,14 +22,15 @@ import { } from '@backstage/types'; import isEqual from 'lodash/isEqual'; import fetch from 'node-fetch'; -import yaml from 'yaml'; import { ConfigTransformer, createConfigTransformer } from './transform'; import { AsyncConfigSourceGenerator, ConfigSource, SubstitutionFunc, ReadConfigDataOptions, + Parser, } from './types'; +import { parseYamlContent } from './utils'; const DEFAULT_RELOAD_INTERVAL = { seconds: 60 }; @@ -55,6 +56,11 @@ export interface RemoteConfigSourceOptions { * A substitution function to use instead of the default environment substitution. */ substitutionFunc?: SubstitutionFunc; + + /** + * A content parsing function to transform string content to configuration values. + */ + parser?: Parser; } /** @@ -84,6 +90,7 @@ export class RemoteConfigSource implements ConfigSource { readonly #url: string; readonly #reloadIntervalMs: number; readonly #transformer: ConfigTransformer; + readonly #parser: Parser; private constructor(options: RemoteConfigSourceOptions) { this.#url = options.url; @@ -93,6 +100,7 @@ export class RemoteConfigSource implements ConfigSource { this.#transformer = createConfigTransformer({ substitutionFunc: options.substitutionFunc, }); + this.#parser = options.parser ?? parseYamlContent; } async *readConfigData( @@ -135,11 +143,21 @@ export class RemoteConfigSource implements ConfigSource { throw await ResponseError.fromResponse(res); } - const content = await res.text(); - const data = await this.#transformer(yaml.parse(content)); - if (data === null) { + const contents = await res.text(); + const { result: rawData } = await this.#parser({ contents }); + if (rawData === undefined) { + /** + * This error message is/was coupled to the implementation and with refactoring it is no longer truly accurate + * This behavior is also inconsistent with {@link FileConfigSource}, which doesn't error on unparseable or empty + * content + * + * Preserving to not make a breaking change + */ throw new Error('configuration data is null'); - } else if (typeof data !== 'object') { + } + + const data = await this.#transformer(rawData); + if (typeof data !== 'object') { throw new Error('configuration data is not an object'); } else if (Array.isArray(data)) { throw new Error( diff --git a/packages/config-loader/src/sources/index.ts b/packages/config-loader/src/sources/index.ts index a25f689b40..3bf8e71990 100644 --- a/packages/config-loader/src/sources/index.ts +++ b/packages/config-loader/src/sources/index.ts @@ -38,4 +38,5 @@ export type { ConfigSourceData, ReadConfigDataOptions, AsyncConfigSourceGenerator, + Parser, } from './types'; diff --git a/packages/config-loader/src/sources/types.ts b/packages/config-loader/src/sources/types.ts index dec6f415f2..abdf2f5c8e 100644 --- a/packages/config-loader/src/sources/types.ts +++ b/packages/config-loader/src/sources/types.ts @@ -15,6 +15,7 @@ */ import { AppConfig } from '@backstage/config'; +import { JsonObject } from '@backstage/types'; /** * The data returned by {@link ConfigSource.readConfigData}. @@ -89,3 +90,18 @@ export interface ConfigSource { * @public */ export type SubstitutionFunc = (name: string) => Promise; + +/** + * A custom function to be used for parsing configuration content. + * + * @remarks + * + * The default parsing function will parse configuration content as yaml. + * + * @public + */ +export type Parser = ({ + contents, +}: { + contents: string; +}) => Promise<{ result?: JsonObject }>; diff --git a/packages/config-loader/src/sources/utils.ts b/packages/config-loader/src/sources/utils.ts index 9978234195..b948cc47b4 100644 --- a/packages/config-loader/src/sources/utils.ts +++ b/packages/config-loader/src/sources/utils.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import { Parser } from './types'; +import yaml from 'yaml'; + /** @internal */ export interface SimpleDeferred { promise: Promise; @@ -55,3 +58,9 @@ export async function waitOrAbort( signals.forEach(s => s.addEventListener('abort', onAbort)); }); } + +/** @internal */ +export const parseYamlContent: Parser = async ({ contents }) => { + const parsed = yaml.parse(contents); + return { result: parsed === null ? undefined : parsed }; +};