feat: add parsing configuration option for ConfigSources

FileConfigSource and RemoteConfigSource can now be optionally configured to operate on more than yaml files

Signed-off-by: Tim Klever <tim.v.klever@aexp.com>

.changeset/young-peaches-shake.md

@@ -0,0 +1,5 @@
+---
+'@backstage/config-loader': minor
+---
+
+Add configuration key to File and Remote `ConfigSource`s that enables configuration of parsing logic. Previously limited to yaml, these `ConfigSource`s now allow for a multitude of parsing options (e.g. JSON).

packages/config-loader/api-report.md

@@ -141,6 +141,7 @@ export class FileConfigSource implements ConfigSource {
 
 // @public
 export interface FileConfigSourceOptions {
+  parser?: Parser;
   path: string;
   substitutionFunc?: EnvFunc;
   watch?: boolean;
@@ -218,6 +219,11 @@ export interface MutableConfigSourceOptions {
   data?: JsonObject;
 }
 
+// @public
+export type Parser = ({ contents }: { contents: string }) => Promise<{
+  result?: JsonObject;
+}>;
+
 // @public
 export interface ReadConfigDataOptions {
   // (undocumented)
@@ -242,6 +248,7 @@ export class RemoteConfigSource implements ConfigSource {
 
 // @public
 export interface RemoteConfigSourceOptions {
+  parser?: Parser;
   reloadInterval?: HumanDuration;
   substitutionFunc?: EnvFunc;
   url: string;

packages/config-loader/src/sources/FileConfigSource.test.ts

@@ -59,6 +59,19 @@ describe('FileConfigSource', () => {
     ]);
   });
 
+  it('should read a config file with optional parser', async () => {
+    const tmp = await tmpFiles({ 'a.json': JSON.stringify({ a: 1 }) });
+
+    const source = FileConfigSource.create({
+      path: tmp.resolve('a.json'),
+      parser: async ({ contents }) => ({ result: JSON.parse(contents) }),
+    });
+
+    await expect(readN(source, 1)).resolves.toEqual([
+      [{ data: { a: 1 }, context: 'a.json', path: tmp.resolve('a.json') }],
+    ]);
+  });
+
   it('should watch config files', async () => {
     const tmp = await tmpFiles({ 'a.yaml': 'a: 1' });
 

packages/config-loader/src/sources/FileConfigSource.ts

@@ -17,16 +17,17 @@
 import chokidar, { FSWatcher } from 'chokidar';
 import fs from 'fs-extra';
 import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path';
-import yaml from 'yaml';
 import {
   AsyncConfigSourceGenerator,
   ConfigSource,
   ConfigSourceData,
   SubstitutionFunc,
+  Parser,
   ReadConfigDataOptions,
 } from './types';
 import { createConfigTransformer } from './transform';
 import { NotFoundError } from '@backstage/errors';
+import { parseYamlContent } from './utils';
 
 /**
  * Options for {@link FileConfigSource.create}.
@@ -48,6 +49,11 @@ export interface FileConfigSourceOptions {
    * A substitution function to use instead of the default environment substitution.
    */
   substitutionFunc?: SubstitutionFunc;
+
+  /**
+   * A content parsing function to transform string content to configuration values.
+   */
+  parser?: Parser;
 }
 
 async function readFile(path: string): Promise<string | undefined> {
@@ -95,11 +101,13 @@ export class FileConfigSource implements ConfigSource {
   readonly #path: string;
   readonly #substitutionFunc?: SubstitutionFunc;
   readonly #watch?: boolean;
+  readonly #parser: Parser;
 
   private constructor(options: FileConfigSourceOptions) {
     this.#path = options.path;
     this.#substitutionFunc = options.substitutionFunc;
     this.#watch = options.watch ?? true;
+    this.#parser = options.parser ?? parseYamlContent;
   }
 
   // Work is duplicated across each read, in practice that should not
@@ -154,12 +162,12 @@ export class FileConfigSource implements ConfigSource {
         watchedPaths.push(this.#path);
       }
 
-      const content = await readFile(this.#path);
-      if (content === undefined) {
+      const contents = await readFile(this.#path);
+      if (contents === undefined) {
         throw new NotFoundError(`Config file "${this.#path}" does not exist`);
       }
-      const parsed = yaml.parse(content);
-      if (parsed === null) {
+      const { result: parsed } = await this.#parser({ contents });
+      if (parsed === undefined) {
         return [];
       }
       try {

packages/config-loader/src/sources/RemoteConfigSource.test.ts

@@ -59,6 +59,45 @@ app:
     ]);
   });
 
+  it('should load and parse config from a remote URL', async () => {
+    worker.use(
+      rest.get('http://localhost/config.json', (_req, res, ctx) =>
+        res(
+          ctx.body(
+            JSON.stringify({
+              app: {
+                title: 'Example App',
+                substituted: 'x',
+                escaped: '$${VALUE}',
+              },
+            }),
+          ),
+        ),
+      ),
+    );
+
+    const source = RemoteConfigSource.create({
+      url: 'http://localhost/config.json',
+      substitutionFunc: async () => 'x',
+      parser: async ({ contents }) => ({ result: JSON.parse(contents) }),
+    });
+
+    await expect(readN(source, 1)).resolves.toEqual([
+      [
+        {
+          context: 'http://localhost/config.json',
+          data: {
+            app: {
+              title: 'Example App',
+              substituted: 'x',
+              escaped: '${VALUE}',
+            },
+          },
+        },
+      ],
+    ]);
+  });
+
   it('should reload config from a remote URL', async () => {
     let fetched = false;
 

packages/config-loader/src/sources/RemoteConfigSource.ts

@@ -22,14 +22,15 @@ import {
 } from '@backstage/types';
 import isEqual from 'lodash/isEqual';
 import fetch from 'node-fetch';
-import yaml from 'yaml';
 import { ConfigTransformer, createConfigTransformer } from './transform';
 import {
   AsyncConfigSourceGenerator,
   ConfigSource,
   SubstitutionFunc,
   ReadConfigDataOptions,
+  Parser,
 } from './types';
+import { parseYamlContent } from './utils';
 
 const DEFAULT_RELOAD_INTERVAL = { seconds: 60 };
 
@@ -55,6 +56,11 @@ export interface RemoteConfigSourceOptions {
    * A substitution function to use instead of the default environment substitution.
    */
   substitutionFunc?: SubstitutionFunc;
+
+  /**
+   * A content parsing function to transform string content to configuration values.
+   */
+  parser?: Parser;
 }
 
 /**
@@ -84,6 +90,7 @@ export class RemoteConfigSource implements ConfigSource {
   readonly #url: string;
   readonly #reloadIntervalMs: number;
   readonly #transformer: ConfigTransformer;
+  readonly #parser: Parser;
 
   private constructor(options: RemoteConfigSourceOptions) {
     this.#url = options.url;
@@ -93,6 +100,7 @@ export class RemoteConfigSource implements ConfigSource {
     this.#transformer = createConfigTransformer({
       substitutionFunc: options.substitutionFunc,
     });
+    this.#parser = options.parser ?? parseYamlContent;
   }
 
   async *readConfigData(
@@ -135,11 +143,21 @@ export class RemoteConfigSource implements ConfigSource {
       throw await ResponseError.fromResponse(res);
     }
 
-    const content = await res.text();
-    const data = await this.#transformer(yaml.parse(content));
-    if (data === null) {
+    const contents = await res.text();
+    const { result: rawData } = await this.#parser({ contents });
+    if (rawData === undefined) {
+      /**
+       * This error message is/was coupled to the implementation and with refactoring it is no longer truly accurate
+       * This behavior is also inconsistent with {@link FileConfigSource}, which doesn't error on unparseable or empty
+       * content
+       *
+       * Preserving to not make a breaking change
+       */
       throw new Error('configuration data is null');
-    } else if (typeof data !== 'object') {
+    }
+
+    const data = await this.#transformer(rawData);
+    if (typeof data !== 'object') {
       throw new Error('configuration data is not an object');
     } else if (Array.isArray(data)) {
       throw new Error(

packages/config-loader/src/sources/index.ts

@@ -38,4 +38,5 @@ export type {
   ConfigSourceData,
   ReadConfigDataOptions,
   AsyncConfigSourceGenerator,
+  Parser,
 } from './types';

packages/config-loader/src/sources/types.ts

@@ -15,6 +15,7 @@
  */
 
 import { AppConfig } from '@backstage/config';
+import { JsonObject } from '@backstage/types';
 
 /**
  * The data returned by {@link ConfigSource.readConfigData}.
@@ -89,3 +90,18 @@ export interface ConfigSource {
  * @public
  */
 export type SubstitutionFunc = (name: string) => Promise<string | undefined>;
+
+/**
+ * A custom function to be used for parsing configuration content.
+ *
+ * @remarks
+ *
+ * The default parsing function will parse configuration content as yaml.
+ *
+ * @public
+ */
+export type Parser = ({
+  contents,
+}: {
+  contents: string;
+}) => Promise<{ result?: JsonObject }>;

packages/config-loader/src/sources/utils.ts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+import { Parser } from './types';
+import yaml from 'yaml';
+
 /** @internal */
 export interface SimpleDeferred<T> {
   promise: Promise<T>;
@@ -55,3 +58,9 @@ export async function waitOrAbort<T>(
     signals.forEach(s => s.addEventListener('abort', onAbort));
   });
 }
+
+/** @internal */
+export const parseYamlContent: Parser = async ({ contents }) => {
+  const parsed = yaml.parse(contents);
+  return { result: parsed === null ? undefined : parsed };
+};