paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 53 Packages Projects Releases Wiki Activity

Merge pull request #17209 from backstage/rugvip/confload

Browse Source 
config-loader: refactor config loading around a new ConfigSource interface
This commit is contained in:
Patrik Oldsberg
2023-04-24 12:07:02 +02:00
committed by GitHub
parent e0506af8fc d384c83a12
commit a6702d9f1a
53 changed files with 3003 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/modern-pumpkins-join.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-app-api': patch
---
Switch `configServiceFactory` to use `ConfigSources` from `@backstage/config-loader` to load config.
.changeset/moody-lizards-beam.md
+40
View File
@@ -0,0 +1,40 @@
---
'@backstage/config-loader': minor
---
Introduced a new config source system to replace `loadConfig`. There is a new `ConfigSource` interface along with utilities provided by `ConfigSources`, as well as a number of built-in configuration source implementations. The new system is more flexible and makes it easier to create new and reusable sources of configuration, such as loading configuration from secret providers.
The following is an example of how to load configuration using the default behavior:
```ts
const source = ConfigSources.default({
argv: options?.argv,
remote: options?.remote,
});
const config = await ConfigSources.toConfig(source);
```
The `ConfigSource` interface looks like this:
```ts
export interface ConfigSource {
readConfigData(options?: ReadConfigDataOptions): AsyncConfigSourceIterator;
}
```
It is best implemented using an async iterator:
```ts
class MyConfigSource implements ConfigSource {
async *readConfigData() {
yield {
config: [
{
context: 'example',
data: { backend: { baseUrl: 'http://localhost' } },
},
],
};
}
}
```
packages/backend-app-api/api-report.md
+2 -1
View File
@@ -26,6 +26,7 @@ import { LoggerService } from '@backstage/backend-plugin-api';
import { PermissionsService } from '@backstage/backend-plugin-api';
import { PluginDatabaseManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { RemoteConfigSourceOptions } from '@backstage/config-loader';
import { RequestHandler } from 'express';
import { RequestListener } from 'http';
import { RootHttpRouterService } from '@backstage/backend-plugin-api';
@@ -54,7 +55,7 @@ export const cacheServiceFactory: () => ServiceFactory<CacheClient, 'plugin'>;
// @public (undocumented)
export interface ConfigFactoryOptions {
argv?: string[];
remote?: LoadConfigOptionsRemote;
remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
}
// @public (undocumented)
packages/backend-app-api/src/services/implementations/config/configServiceFactory.ts
+12 -7
View File
@@ -18,8 +18,10 @@ import {
coreServices,
createServiceFactory,
} from '@backstage/backend-plugin-api';
import { LoadConfigOptionsRemote } from '@backstage/config-loader';
import { loadBackendConfig } from '../../../config';
import {
ConfigSources,
RemoteConfigSourceOptions,
} from '@backstage/config-loader';
/** @public */
export interface ConfigFactoryOptions {
@@ -31,7 +33,7 @@ export interface ConfigFactoryOptions {
/**
* Enables and sets options for remote configuration loading.
*/
remote?: LoadConfigOptionsRemote;
remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
}
/** @public */
@@ -39,10 +41,13 @@ export const configServiceFactory = createServiceFactory(
(options?: ConfigFactoryOptions) => ({
service: coreServices.config,
deps: {},
async factory({}) {
const { argv = process.argv, remote } = options ?? {};
const { config } = await loadBackendConfig({ argv, remote });
return config;
async factory() {
const source = ConfigSources.default({
argv: options?.argv,
remote: options?.remote,
});
console.log(`Loading config from ${source}`);
return await ConfigSources.toConfig(source);
},
}),
);
packages/config-loader/api-report.md
+188 -5
View File
@@ -4,8 +4,45 @@
```ts
import { AppConfig } from '@backstage/config';
import { Config } from '@backstage/config';
import { HumanDuration } from '@backstage/types';
import { JsonObject } from '@backstage/types';
import { JSONSchema7 } from 'json-schema';
import { Observable } from '@backstage/types';
// @public
export interface AsyncConfigSourceIterator
extends AsyncIterator<
{
configs: ConfigSourceData[];
},
void,
void
> {
// (undocumented)
[Symbol.asyncIterator](): AsyncIterator<
{
configs: ConfigSourceData[];
},
void,
void
>;
}
// @public
export interface BaseConfigSourcesOptions {
// (undocumented)
remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
// (undocumented)
rootDir?: string;
// (undocumented)
substitutionFunc?: EnvFunc;
}
// @public
export interface ClosableConfig extends Config {
close(): void;
}
// @public
export type ConfigSchema = {
@@ -25,7 +62,55 @@ export type ConfigSchemaProcessingOptions = {
withDeprecatedKeys?: boolean;
};
// @public (undocumented)
// @public
export interface ConfigSource {
// (undocumented)
readConfigData(options?: ReadConfigDataOptions): AsyncConfigSourceIterator;
}
// @public
export interface ConfigSourceData extends AppConfig {
path?: string;
}
// @public
export class ConfigSources {
static default(options: ConfigSourcesDefaultOptions): ConfigSource;
static defaultForTargets(
options: ConfigSourcesDefaultForTargetsOptions,
): ConfigSource;
static merge(sources: ConfigSource[]): ConfigSource;
static parseArgs(argv?: string[]): ConfigSourceTarget[];
static toConfig(source: ConfigSource): Promise<ClosableConfig>;
}
// @public
export interface ConfigSourcesDefaultForTargetsOptions
extends BaseConfigSourcesOptions {
// (undocumented)
targets: ConfigSourceTarget[];
}
// @public
export interface ConfigSourcesDefaultOptions extends BaseConfigSourcesOptions {
// (undocumented)
argv?: string[];
// (undocumented)
env?: Record<string, string>;
}
// @public
export type ConfigSourceTarget =
| {
type: 'path';
target: string;
}
| {
type: 'url';
target: string;
};
// @public @deprecated (undocumented)
export type ConfigTarget =
| {
path: string;
@@ -38,11 +123,43 @@ export type ConfigTarget =
export type ConfigVisibility = 'frontend' | 'backend' | 'secret';
// @public
export class EnvConfigSource implements ConfigSource {
static create(options: EnvConfigSourceOptions): ConfigSource;
// (undocumented)
readConfigData(): AsyncConfigSourceIterator;
// (undocumented)
toString(): string;
}
// @public
export interface EnvConfigSourceOptions {
env?: Record<string, string | undefined>;
}
// @public
export type EnvFunc = (name: string) => Promise<string | undefined>;
// @public
export class FileConfigSource implements ConfigSource {
static create(options: FileConfigSourceOptions): ConfigSource;
// (undocumented)
readConfigData(options?: ReadConfigDataOptions): AsyncConfigSourceIterator;
// (undocumented)
toString(): string;
}
// @public
export interface FileConfigSourceOptions {
path: string;
substitutionFunc?: EnvFunc;
}
// @public @deprecated
export function loadConfig(
options: LoadConfigOptions,
): Promise<LoadConfigResult>;
// @public
// @public @deprecated
export type LoadConfigOptions = {
configRoot: string;
configTargets: ConfigTarget[];
@@ -51,18 +168,18 @@ export type LoadConfigOptions = {
watch?: LoadConfigOptionsWatch;
};
// @public (undocumented)
// @public @deprecated (undocumented)
export type LoadConfigOptionsRemote = {
reloadIntervalSeconds: number;
};
// @public (undocumented)
// @public @deprecated (undocumented)
export type LoadConfigOptionsWatch = {
onChange: (configs: AppConfig[]) => void;
stopSignal?: Promise<void>;
};
// @public
// @public @deprecated
export type LoadConfigResult = {
appConfigs: AppConfig[];
};
@@ -86,10 +203,76 @@ export type LoadConfigSchemaOptions =
export function mergeConfigSchemas(schemas: JSONSchema7[]): JSONSchema7;
// @public
export class MutableConfigSource implements ConfigSource {
close(): void;
static create(options?: MutableConfigSourceOptions): MutableConfigSource;
// (undocumented)
readConfigData(
options?: ReadConfigDataOptions | undefined,
): AsyncConfigSourceIterator;
setData(data: JsonObject): void;
// (undocumented)
toString(): string;
}
// @public
export interface MutableConfigSourceOptions {
// (undocumented)
context?: string;
// (undocumented)
data?: JsonObject;
}
// @public
export interface ReadConfigDataOptions {
// (undocumented)
signal?: AbortSignal;
}
// @public @deprecated
export function readEnvConfig(env: {
[name: string]: string | undefined;
}): AppConfig[];
// @public
export class RemoteConfigSource implements ConfigSource {
static create(options: RemoteConfigSourceOptions): ConfigSource;
// (undocumented)
readConfigData(
options?: ReadConfigDataOptions | undefined,
): AsyncConfigSourceIterator;
// (undocumented)
toString(): string;
}
// @public
export interface RemoteConfigSourceOptions {
reloadInterval?: HumanDuration;
substitutionFunc?: EnvFunc;
url: string;
}
// @public
export class StaticConfigSource implements ConfigSource {
static create(options: StaticConfigSourceOptions): ConfigSource;
// (undocumented)
readConfigData(): AsyncConfigSourceIterator;
// (undocumented)
toString(): string;
}
// @public
export interface StaticConfigSourceOptions {
// (undocumented)
context?: string;
// (undocumented)
data:
| JsonObject
| Observable<JsonObject>
| PromiseLike<JsonObject>
| AsyncIterable<JsonObject>;
}
// @public
export type TransformFunc<T extends number | string | boolean> = (
value: T,
packages/config-loader/package.json
+5 -1
View File
@@ -44,19 +44,23 @@
"json-schema": "^0.4.0",
"json-schema-merge-allof": "^0.8.1",
"json-schema-traverse": "^1.0.0",
"lodash": "^4.17.21",
"minimist": "^1.2.5",
"node-fetch": "^2.6.7",
"typescript-json-schema": "^0.55.0",
"yaml": "^2.0.0",
"yup": "^0.32.9"
},
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/json-schema-merge-allof": "^0.6.0",
"@types/mock-fs": "^4.10.0",
"@types/node": "^16.11.26",
"@types/yup": "^0.29.13",
"mock-fs": "^5.1.0",
"msw": "^1.0.0"
"msw": "^1.0.0",
"zen-observable": "^0.10.0"
},
"files": [
"dist"
packages/config-loader/src/index.ts
+3 -2
View File
@@ -20,14 +20,14 @@
* @packageDocumentation
*/
export { readEnvConfig, loadConfigSchema, mergeConfigSchemas } from './lib';
export { loadConfigSchema, mergeConfigSchemas } from './schema';
export type {
ConfigSchema,
ConfigSchemaProcessingOptions,
ConfigVisibility,
LoadConfigSchemaOptions,
TransformFunc,
} from './lib';
} from './schema';
export { loadConfig } from './loader';
export type {
ConfigTarget,
@@ -36,3 +36,4 @@ export type {
LoadConfigOptionsRemote,
LoadConfigResult,
} from './loader';
export * from './sources';
packages/config-loader/src/lib/index.ts
-20
View File
@@ -1,20 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { readEnvConfig } from './env';
export * from './transform';
export * from './schema';
export { isValidUrl } from './urls';
packages/config-loader/src/lib/urls.test.ts
-34
View File
@@ -1,34 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { isValidUrl } from './urls';
describe('isValidUrl', () => {
it('should return true for url', () => {
const validUrl = isValidUrl('http://some.valid.url');
expect(validUrl).toBe(true);
});
it('should return false for absolute path', () => {
const validUrl = isValidUrl('/some/absolute/path');
expect(validUrl).toBe(false);
});
it('should return false for relative path', () => {
const validUrl = isValidUrl('../some/relative/path');
expect(validUrl).toBe(false);
});
});
packages/config-loader/src/lib/urls.ts
-25
View File
@@ -1,25 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function isValidUrl(url: string): boolean {
try {
// eslint-disable-next-line no-new
new URL(url);
return true;
} catch {
return false;
}
}
packages/config-loader/src/loader.test.ts
+11
View File
@@ -132,6 +132,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config.yaml',
},
],
});
@@ -186,6 +187,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config.yaml',
},
{
context: 'app-config2.yaml',
@@ -196,6 +198,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config2.yaml',
},
],
});
@@ -218,6 +221,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config.yaml',
},
],
});
@@ -243,6 +247,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config.yaml',
},
{
context: 'app-config.development.yaml',
@@ -259,6 +264,7 @@ describe('loadConfig', () => {
secret: 'abc123',
},
},
path: '/root/app-config.development.yaml',
},
],
});
@@ -282,6 +288,7 @@ describe('loadConfig', () => {
noSubstitute: 'notSubstituted',
},
},
path: '/root/app-config.substitute.yaml',
},
],
});
@@ -311,6 +318,7 @@ describe('loadConfig', () => {
escaped: '${Escaped}',
},
},
path: '/root/app-config.yaml',
},
],
});
@@ -328,6 +336,7 @@ describe('loadConfig', () => {
title: 'New Title',
},
},
path: '/root/app-config.yaml',
},
]);
@@ -364,6 +373,7 @@ describe('loadConfig', () => {
secret: 'abc123',
},
},
path: '/root/app-config.development.yaml',
},
],
});
@@ -388,6 +398,7 @@ describe('loadConfig', () => {
secret: 'abc234',
},
},
path: '/root/app-config.development.yaml',
},
]);
packages/config-loader/src/loader.ts
+53 -226
View File
@@ -14,25 +14,19 @@
* limitations under the License.
*/
import fs from 'fs-extra';
import yaml from 'yaml';
import chokidar from 'chokidar';
import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path';
import { AppConfig } from '@backstage/config';
import { ForwardedError } from '@backstage/errors';
import {
applyConfigTransforms,
createIncludeTransform,
createSubstitutionTransform,
isValidUrl,
readEnvConfig,
} from './lib';
import fetch from 'node-fetch';
import { ConfigSources } from './sources';
/** @public */
/**
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export type ConfigTarget = { path: string } | { url: string };
/** @public */
/**
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export type LoadConfigOptionsWatch = {
/**
* A listener that is called when a config file is changed.
@@ -45,7 +39,10 @@ export type LoadConfigOptionsWatch = {
stopSignal?: Promise<void>;
};
/** @public */
/**
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export type LoadConfigOptionsRemote = {
/**
* A remote config reloading period, in seconds
@@ -57,6 +54,7 @@ export type LoadConfigOptionsRemote = {
* Options that control the loading of configuration files in the backend.
*
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export type LoadConfigOptions = {
// The root directory of the config loading context. Used to find default configs.
@@ -86,6 +84,7 @@ export type LoadConfigOptions = {
/**
* Results of loading configuration files.
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export type LoadConfigResult = {
/**
@@ -98,227 +97,55 @@ export type LoadConfigResult = {
* Load configuration data.
*
* @public
* @deprecated Use {@link ConfigSources.default} instead.
*/
export async function loadConfig(
options: LoadConfigOptions,
): Promise<LoadConfigResult> {
const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options;
const source = ConfigSources.default({
substitutionFunc: options.experimentalEnvFunc,
remote: options.remote && {
reloadInterval: { seconds: options.remote.reloadIntervalSeconds },
},
rootDir: options.configRoot,
argv: options.configTargets.flatMap(t => [
'--config',
'url' in t ? t.url : t.path,
]),
});
const configPaths: string[] = options.configTargets
.slice()
.filter((e): e is { path: string } => e.hasOwnProperty('path'))
.map(configTarget => configTarget.path);
return new Promise<LoadConfigResult>((resolve, reject) => {
async function loadConfigReaderLoop() {
let loaded = false;
const configUrls: string[] = options.configTargets
.slice()
.filter((e): e is { url: string } => e.hasOwnProperty('url'))
.map(configTarget => configTarget.url);
if (remote === undefined) {
if (configUrls.length > 0) {
throw new Error(
`Please make sure you are passing the remote option when loading remote configurations. See https://backstage.io/docs/conf/writing#configuration-files for detailed info.`,
);
}
} else if (remote.reloadIntervalSeconds <= 0) {
throw new Error(
`Remote config must be contain a non zero reloadIntervalSeconds: <seconds> value`,
);
}
// If no paths are provided, we default to reading
// `app-config.yaml` and, if it exists, `app-config.local.yaml`
if (configPaths.length === 0 && configUrls.length === 0) {
configPaths.push(resolvePath(configRoot, 'app-config.yaml'));
const localConfig = resolvePath(configRoot, 'app-config.local.yaml');
if (await fs.pathExists(localConfig)) {
configPaths.push(localConfig);
}
}
const env = envFunc ?? (async (name: string) => process.env[name]);
const loadConfigFiles = async () => {
const fileConfigs = [];
const loadedPaths = new Set<string>();
for (const configPath of configPaths) {
if (!isAbsolute(configPath)) {
throw new Error(`Config load path is not absolute: '${configPath}'`);
}
const dir = dirname(configPath);
const readFile = (path: string) => {
const fullPath = resolvePath(dir, path);
// if we read a file when building configuration,
// we should include that file when watching for
// changes, too.
loadedPaths.add(fullPath);
return fs.readFile(fullPath, 'utf8');
};
const input = yaml.parse(await readFile(configPath));
// A completely empty file ends up as a null return value
if (input !== null) {
const substitutionTransform = createSubstitutionTransform(env);
const data = await applyConfigTransforms(dir, input, [
createIncludeTransform(env, readFile, substitutionTransform),
substitutionTransform,
]);
fileConfigs.push({ data, context: basename(configPath) });
}
}
return { fileConfigs, loadedPaths };
};
const loadRemoteConfigFiles = async () => {
const configs: AppConfig[] = [];
const readConfigFromUrl = async (url: string) => {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Could not read config file at ${url}`);
}
return await response.text();
};
for (let i = 0; i < configUrls.length; i++) {
const configUrl = configUrls[i];
if (!isValidUrl(configUrl)) {
throw new Error(`Config load path is not valid: '${configUrl}'`);
}
const remoteConfigContent = await readConfigFromUrl(configUrl);
if (!remoteConfigContent) {
throw new Error(`Config is not valid`);
}
const configYaml = yaml.parse(remoteConfigContent);
const substitutionTransform = createSubstitutionTransform(env);
const data = await applyConfigTransforms(configRoot, configYaml, [
substitutionTransform,
]);
configs.push({ data, context: configUrl });
}
return configs;
};
let fileConfigs: AppConfig[];
let loadedPaths: Set<string>;
try {
({ fileConfigs, loadedPaths } = await loadConfigFiles());
} catch (error) {
throw new ForwardedError('Failed to read static configuration file', error);
}
let remoteConfigs: AppConfig[] = [];
if (remote) {
try {
remoteConfigs = await loadRemoteConfigFiles();
} catch (error) {
throw new ForwardedError(
`Failed to read remote configuration file`,
error,
);
}
}
const envConfigs = readEnvConfig(process.env);
const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => {
let watchedFiles = Array.from(loadedPaths);
const watcher = chokidar.watch(watchedFiles, {
usePolling: process.env.NODE_ENV === 'test',
});
let currentSerializedConfig = JSON.stringify(fileConfigs);
watcher.on('change', async () => {
try {
const { fileConfigs: newConfigs, loadedPaths: newLoadedPaths } =
await loadConfigFiles();
const abortController = new AbortController();
options.watch?.stopSignal?.then(() => abortController.abort());
// Replace watches to handle any added or removed
// $include or $file expressions.
watcher.unwatch(watchedFiles);
watchedFiles = Array.from(newLoadedPaths);
watcher.add(watchedFiles);
for await (const { configs } of source.readConfigData({
signal: abortController.signal,
})) {
if (loaded) {
options.watch?.onChange(configs);
} else {
resolve({ appConfigs: configs });
loaded = true;
const newSerializedConfig = JSON.stringify(newConfigs);
if (currentSerializedConfig === newSerializedConfig) {
return;
if (options.watch) {
options.watch.stopSignal?.then(() => abortController.abort());
} else {
abortController.abort();
}
}
}
currentSerializedConfig = newSerializedConfig;
watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]);
} catch (error) {
console.error(`Failed to reload configuration files, ${error}`);
if (loaded) {
console.error(`Failed to reload configuration, ${error}`);
} else {
reject(error);
}
}
});
if (watchProp.stopSignal) {
watchProp.stopSignal.then(() => {
watcher.close();
});
}
};
const watchRemoteConfig = (
watchProp: LoadConfigOptionsWatch,
remoteProp: LoadConfigOptionsRemote,
) => {
const hasConfigChanged = async (
oldRemoteConfigs: AppConfig[],
newRemoteConfigs: AppConfig[],
) => {
return (
JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs)
);
};
let handle: NodeJS.Timeout | undefined;
try {
handle = setInterval(async () => {
const newRemoteConfigs = await loadRemoteConfigFiles();
if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) {
remoteConfigs = newRemoteConfigs;
watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]);
}
}, remoteProp.reloadIntervalSeconds * 1000);
} catch (error) {
console.error(`Failed to reload configuration files, ${error}`);
}
if (watchProp.stopSignal) {
watchProp.stopSignal.then(() => {
if (handle !== undefined) {
clearInterval(handle);
handle = undefined;
}
});
}
};
// Set up config file watching if requested by the caller
if (watch) {
watchConfigFile(watch);
}
if (watch && remote) {
watchRemoteConfig(watch, remote);
}
return {
appConfigs: remote
? [...remoteConfigs, ...fileConfigs, ...envConfigs]
: [...fileConfigs, ...envConfigs],
};
loadConfigReaderLoop();
});
}
packages/config-loader/src/lib/schema/collect.test.ts → packages/config-loader/src/schema/collect.test.ts
View File
packages/config-loader/src/lib/schema/collect.ts → packages/config-loader/src/schema/collect.ts
View File
packages/config-loader/src/lib/schema/compile.test.ts → packages/config-loader/src/schema/compile.test.ts
View File
packages/config-loader/src/lib/schema/compile.ts → packages/config-loader/src/schema/compile.ts
View File
packages/config-loader/src/lib/schema/filtering.test.ts → packages/config-loader/src/schema/filtering.test.ts
View File
packages/config-loader/src/lib/schema/filtering.ts → packages/config-loader/src/schema/filtering.ts
View File
packages/config-loader/src/lib/schema/index.ts → packages/config-loader/src/schema/index.ts
View File
packages/config-loader/src/lib/schema/load.test.ts → packages/config-loader/src/schema/load.test.ts
View File
packages/config-loader/src/lib/schema/load.ts → packages/config-loader/src/schema/load.ts
View File
packages/config-loader/src/lib/schema/types.ts → packages/config-loader/src/schema/types.ts
View File
packages/config-loader/src/sources/ConfigSources.test.ts
+234
View File
@@ -0,0 +1,234 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { ConfigSources } from './ConfigSources';
import { ConfigSource } from './types';
import { MutableConfigSource } from './MutableConfigSource';
jest.mock('./FileConfigSource', () => ({
FileConfigSource: {
create: (opts: {}) => ({ ...opts, name: 'FileConfigSource' }),
},
}));
jest.mock('./RemoteConfigSource', () => ({
RemoteConfigSource: {
create: (opts: {}) => ({ ...opts, name: 'RemoteConfigSource' }),
},
}));
jest.mock('./EnvConfigSource', () => ({
EnvConfigSource: {
create: (opts: {}) => ({ ...opts, name: 'EnvConfigSource' }),
},
}));
function mergeSources(source: ConfigSource): ConfigSource[] {
return (source as any)[
Symbol.for('@backstage/config-loader#MergedConfigSource.sources')
] as ConfigSource[];
}
describe('ConfigSources', () => {
it('should parse args', () => {
expect(ConfigSources.parseArgs([])).toEqual([]);
expect(
ConfigSources.parseArgs(['--config', 'a.yaml', '--config=b.yaml']),
).toEqual([
{ type: 'path', target: 'a.yaml' },
{ type: 'path', target: 'b.yaml' },
]);
expect(
ConfigSources.parseArgs([
'--config',
'a.yaml',
'--config',
'http://example.com/config.yaml',
]),
).toEqual([
{ type: 'path', target: 'a.yaml' },
{ type: 'url', target: 'http://example.com/config.yaml' },
]);
});
it('should create default sources for targets', () => {
expect(
mergeSources(
ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }),
),
).toEqual([{ name: 'FileConfigSource', path: '/app-config.yaml' }]);
const fsSpy = jest.spyOn(fs, 'pathExistsSync').mockReturnValue(true);
expect(
mergeSources(
ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }),
),
).toEqual([
{ name: 'FileConfigSource', path: '/app-config.yaml' },
{ name: 'FileConfigSource', path: '/app-config.local.yaml' },
]);
fsSpy.mockRestore();
expect(
mergeSources(
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [{ type: 'path', target: '/config.yaml' }],
}),
),
).toEqual([{ name: 'FileConfigSource', path: '/config.yaml' }]);
const subFunc = async () => undefined;
expect(
mergeSources(
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [{ type: 'path', target: '/config.yaml' }],
substitutionFunc: subFunc,
}),
),
).toEqual([
{
name: 'FileConfigSource',
path: '/config.yaml',
substitutionFunc: subFunc,
},
]);
expect(() =>
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [{ type: 'url', target: 'http://example.com/config.yaml' }],
}),
).toThrow(
'Config argument "http://example.com/config.yaml" looks like a URL but remote configuration is not enabled. Enable it by passing the `remote` option',
);
expect(
mergeSources(
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [{ type: 'url', target: 'http://example.com/config.yaml' }],
remote: {},
}),
),
).toEqual([
{ name: 'RemoteConfigSource', url: 'http://example.com/config.yaml' },
]);
expect(
mergeSources(
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [{ type: 'url', target: 'http://example.com/config.yaml' }],
remote: { reloadInterval: { minutes: 2 } },
}),
),
).toEqual([
{
name: 'RemoteConfigSource',
url: 'http://example.com/config.yaml',
reloadInterval: { minutes: 2 },
},
]);
});
it('should create a default source', () => {
expect(
mergeSources(
ConfigSources.default({
rootDir: '/',
env: { HOME: '/' },
}),
),
).toEqual([
{ name: 'FileConfigSource', path: '/app-config.yaml' },
{ name: 'EnvConfigSource', env: { HOME: '/' } },
]);
expect(
mergeSources(
ConfigSources.default({
rootDir: '/',
argv: ['--config', 'a.yaml', '--config=b.yaml'],
env: { HOME: '/' },
}),
),
).toEqual([
{ name: 'FileConfigSource', path: 'a.yaml' },
{ name: 'FileConfigSource', path: 'b.yaml' },
{ name: 'EnvConfigSource', env: { HOME: '/' } },
]);
});
it('should merge sources', () => {
expect(
mergeSources(
ConfigSources.merge([
ConfigSources.defaultForTargets({
rootDir: '/',
targets: [
{ type: 'path', target: '/a.yaml' },
{ type: 'path', target: '/b.yaml' },
{ type: 'path', target: '/c.yaml' },
],
}),
]),
),
).toEqual([
{ name: 'FileConfigSource', path: '/a.yaml' },
{ name: 'FileConfigSource', path: '/b.yaml' },
{ name: 'FileConfigSource', path: '/c.yaml' },
]);
});
it('should create an observable config', async () => {
const source = MutableConfigSource.create({ data: { a: 1 } });
const config = await ConfigSources.toConfig(source);
const listener = jest.fn();
const sub = config.subscribe?.(listener);
expect(config.getNumber('a')).toBe(1);
expect(listener).not.toHaveBeenCalled();
source.setData({ a: 2 });
await new Promise<void>(resolve => setTimeout(resolve));
expect(config.getNumber('a')).toBe(2);
expect(listener).toHaveBeenCalledTimes(1);
sub?.unsubscribe();
source.setData({ a: 3 });
await new Promise<void>(resolve => setTimeout(resolve));
expect(config.getNumber('a')).toBe(3);
expect(listener).toHaveBeenCalledTimes(1);
config.close();
source.setData({ a: 4 });
await new Promise<void>(resolve => setTimeout(resolve));
expect(config.getNumber('a')).toBe(3);
});
it('should fail to create config', async () => {
await expect(
ConfigSources.toConfig({
async *readConfigData() {
throw new Error('NOPE');
},
}),
).rejects.toThrow('NOPE');
});
});
packages/config-loader/src/sources/ConfigSources.ts
+268
View File
@@ -0,0 +1,268 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolve as resolvePath } from 'path';
import fs from 'fs-extra';
import { Config, ConfigReader } from '@backstage/config';
import parseArgs from 'minimist';
import { EnvConfigSource } from './EnvConfigSource';
import { FileConfigSource } from './FileConfigSource';
import { MergedConfigSource } from './MergedConfigSource';
import {
RemoteConfigSource,
RemoteConfigSourceOptions,
} from './RemoteConfigSource';
import { ConfigSource, SubstitutionFunc } from './types';
import { ObservableConfigProxy } from './ObservableConfigProxy';
import { findPaths } from '@backstage/cli-common';
/**
* A target to read configuration from.
*
* @public
*/
export type ConfigSourceTarget =
| {
type: 'path';
target: string;
}
| {
type: 'url';
target: string;
};
/**
* A config implementation that can be closed.
*
* @remarks
*
* Closing the configuration instance will stop the reading from the underlying source.
*
* @public
*/
export interface ClosableConfig extends Config {
/**
* Closes the configuration instance.
*
* @remarks
*
* The configuration instance will still be usable after closing, but it will
* no longer be updated with new values from the underlying source.
*/
close(): void;
}
/**
* Common options for the default Backstage configuration sources.
*
* @public
*/
export interface BaseConfigSourcesOptions {
rootDir?: string;
remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
substitutionFunc?: SubstitutionFunc;
}
/**
* Options for {@link ConfigSources.defaultForTargets}.
*
* @public
*/
export interface ConfigSourcesDefaultForTargetsOptions
extends BaseConfigSourcesOptions {
targets: ConfigSourceTarget[];
}
/**
* Options for {@link ConfigSources.default}.
*
* @public
*/
export interface ConfigSourcesDefaultOptions extends BaseConfigSourcesOptions {
argv?: string[];
env?: Record<string, string>;
}
/**
* A collection of utilities for working with and creating {@link ConfigSource}s.
*
* @public
*/
export class ConfigSources {
/**
* Parses command line arguments and returns the config targets.
*
* @param argv - The command line arguments to parse. Defaults to `process.argv`
* @returns A list of config targets
*/
static parseArgs(argv: string[] = process.argv): ConfigSourceTarget[] {
const args: string[] = [parseArgs(argv).config].flat().filter(Boolean);
return args.map(target => {
try {
const url = new URL(target);
// Some file paths are valid relative URLs, so check if the host is empty too
if (!url.host) {
return { type: 'path', target };
}
return { type: 'url', target };
} catch {
return { type: 'path', target };
}
});
}
/**
* Creates the default config sources for the provided targets.
*
* @remarks
*
* This will create {@link FileConfigSource}s and {@link RemoteConfigSource}s
* for the provided targets, and merge them together to a single source.
* If no targets are provided it will fall back to `app-config.yaml` and
* `app-config.local.yaml`.
*
* URL targets are only supported if the `remote` option is provided.
*
* @param options - Options
* @returns A config source for the provided targets
*/
static defaultForTargets(
options: ConfigSourcesDefaultForTargetsOptions,
): ConfigSource {
const rootDir = options.rootDir ?? findPaths(process.cwd()).targetRoot;
const argSources = options.targets.map(arg => {
if (arg.type === 'url') {
if (!options.remote) {
throw new Error(
`Config argument "${arg.target}" looks like a URL but remote configuration is not enabled. Enable it by passing the \`remote\` option`,
);
}
return RemoteConfigSource.create({
url: arg.target,
substitutionFunc: options.substitutionFunc,
reloadInterval: options.remote.reloadInterval,
});
}
return FileConfigSource.create({
path: arg.target,
substitutionFunc: options.substitutionFunc,
});
});
if (argSources.length === 0) {
const defaultPath = resolvePath(rootDir, 'app-config.yaml');
const localPath = resolvePath(rootDir, 'app-config.local.yaml');
argSources.push(
FileConfigSource.create({
path: defaultPath,
substitutionFunc: options.substitutionFunc,
}),
);
if (fs.pathExistsSync(localPath)) {
argSources.push(
FileConfigSource.create({
path: localPath,
substitutionFunc: options.substitutionFunc,
}),
);
}
}
return this.merge(argSources);
}
/**
* Creates the default config source for Backstage.
*
* @remarks
*
* This will read from `app-config.yaml` and `app-config.local.yaml` by
* default, as well as environment variables prefixed with `APP_CONFIG_`.
* If `--config <path|url>` command line arguments are passed, these will
* override the default configuration file paths. URLs are only supported
* if the `remote` option is provided.
*
* @param options - Options
* @returns The default Backstage config source
*/
static default(options: ConfigSourcesDefaultOptions): ConfigSource {
const argSource = this.defaultForTargets({
...options,
targets: this.parseArgs(options.argv),
});
const envSource = EnvConfigSource.create({
env: options.env,
});
return this.merge([argSource, envSource]);
}
/**
* Merges multiple config sources into a single source that reads from all
* sources and concatenates the result.
*
* @param sources - The config sources to merge
* @returns A single config source that concatenates the data from the given sources
*/
static merge(sources: ConfigSource[]): ConfigSource {
return MergedConfigSource.from(sources);
}
/**
* Creates an observable {@link @backstage/config#Config} implementation from a {@link ConfigSource}.
*
* @remarks
*
* If you only want to read the config once you can close the returned config immediately.
*
* @example
*
* ```ts
* const sources = ConfigSources.default(...)
* const config = await ConfigSources.toConfig(source)
* config.close()
* const example = config.getString(...)
* ```
*
* @param source - The config source to read from
* @returns A promise that resolves to a closable config
*/
static toConfig(source: ConfigSource): Promise<ClosableConfig> {
return new Promise(async (resolve, reject) => {
let config: ObservableConfigProxy | undefined = undefined;
try {
const abortController = new AbortController();
for await (const { configs } of source.readConfigData({
signal: abortController.signal,
})) {
if (config) {
config.setConfig(ConfigReader.fromConfigs(configs));
} else {
config = ObservableConfigProxy.create(abortController);
config!.setConfig(ConfigReader.fromConfigs(configs));
resolve(config);
}
}
} catch (error) {
reject(error);
}
});
}
}
packages/config-loader/src/lib/env.test.ts → packages/config-loader/src/sources/EnvConfigSource.test.ts
+26 -1
View File
@@ -14,7 +14,32 @@
* limitations under the License.
*/
import { readEnvConfig } from './env';
import { EnvConfigSource, readEnvConfig } from './EnvConfigSource';
import { ConfigSource, ConfigSourceData } from './types';
async function readAll(source: ConfigSource) {
const entries = new Array<{ configs: ConfigSourceData[] }>();
for await (const item of source.readConfigData()) {
entries.push(item);
}
return entries;
}
describe('EnvConfigSource', () => {
it('should return empty config for empty env', async () => {
const source = EnvConfigSource.create({ env: {} });
await expect(readAll(source)).resolves.toEqual([{ configs: [] }]);
});
it('should forward config values', async () => {
const source = EnvConfigSource.create({ env: { APP_CONFIG_foo: 'bar' } });
await expect(readAll(source)).resolves.toEqual([
{ configs: [{ context: 'env', data: { foo: 'bar' } }] },
]);
});
});
describe('readEnvConfig', () => {
it('should return empty config for empty env', () => {
packages/config-loader/src/lib/env.ts → packages/config-loader/src/sources/EnvConfigSource.ts
+75 -8
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 The Backstage Authors
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,16 +15,26 @@
*/
import { AppConfig } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { assertError } from '@backstage/errors';
const ENV_PREFIX = 'APP_CONFIG_';
// Update the same pattern in config package if this is changed
const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
import { JsonObject } from '@backstage/types';
import { AsyncConfigSourceIterator, ConfigSource } from './types';
/**
* Read runtime configuration from the environment.
* Options for {@link EnvConfigSource.create}.
*
* @public
*/
export interface EnvConfigSourceOptions {
/**
* The environment variables to use, defaults to `process.env`.
*/
env?: Record<string, string | undefined>;
}
/**
* A config source that reads configuration from the environment.
*
* @remarks
*
* Only environment variables prefixed with APP_CONFIG_ will be considered.
*
@@ -43,6 +53,63 @@ const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
*
* @public
*/
export class EnvConfigSource implements ConfigSource {
/**
* Creates a new config source that reads from the environment.
*
* @param options - Options for the config source.
* @returns A new config source that reads from the environment.
*/
static create(options: EnvConfigSourceOptions): ConfigSource {
return new EnvConfigSource(options?.env ?? process.env);
}
private constructor(
private readonly env: { [name: string]: string | undefined },
) {}
async *readConfigData(): AsyncConfigSourceIterator {
const configs = readEnvConfig(this.env);
yield { configs };
return;
}
toString() {
const keys = Object.keys(this.env).filter(key =>
key.startsWith('APP_CONFIG_'),
);
return `EnvConfigSource{count=${keys.length}}`;
}
}
const ENV_PREFIX = 'APP_CONFIG_';
// Update the same pattern in config package if this is changed
const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
/**
* Read runtime configuration from the environment.
*
* @remarks
*
* Only environment variables prefixed with APP_CONFIG_ will be considered.
*
* For each variable, the prefix will be removed, and rest of the key will
* be split by '_'. Each part will then be used as keys to build up a nested
* config object structure. The treatment of the entire environment variable
* is case-sensitive.
*
* The value of the variable should be JSON serialized, as it will be parsed
* and the type will be kept intact. For example "true" and true are treated
* differently, as well as "42" and 42.
*
* For example, to set the config app.title to "My Title", use the following:
*
* APP_CONFIG_app_title='"My Title"'
*
* @public
* @deprecated Use {@link EnvConfigSource} instead
*/
export function readEnvConfig(env: {
[name: string]: string | undefined;
}): AppConfig[] {
packages/config-loader/src/sources/FileConfigSource.test.ts
+212
View File
@@ -0,0 +1,212 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import { resolve as resolvePath } from 'path';
import { FileConfigSource } from './FileConfigSource';
import { readN } from './__testUtils__/testUtils';
const tmpDirs = new Array<string>();
async function tmpFiles(files: Record<string, string>) {
const tmpDir = await fs.mkdtemp(
resolvePath(os.tmpdir(), 'backstage-unit-test-fixture-'),
);
tmpDirs.push(tmpDir);
for (const [name, content] of Object.entries(files)) {
await fs.writeFile(resolvePath(tmpDir, name), content, 'utf8');
}
return {
resolve(...paths: string[]) {
return resolvePath(tmpDir, ...paths);
},
write: async (name: string, content: string) => {
await fs.writeFile(resolvePath(tmpDir, name), content, 'utf8');
},
};
}
describe('FileConfigSource', () => {
afterEach(async () => {
for (const tmpDir of tmpDirs) {
await fs.remove(tmpDir);
}
});
it('should read a config file', async () => {
const tmp = await tmpFiles({ 'a.yaml': 'a: 1' });
const source = FileConfigSource.create({ path: tmp.resolve('a.yaml') });
await expect(readN(source, 1)).resolves.toEqual([
[{ data: { a: 1 }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should watch config files', async () => {
const tmp = await tmpFiles({ 'a.yaml': 'a: 1' });
const source = FileConfigSource.create({ path: tmp.resolve('a.yaml') });
setTimeout(() => {
tmp.write('a.yaml', 'a: 2');
}, 10);
await expect(readN(source, 2)).resolves.toEqual([
[{ data: { a: 1 }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
[{ data: { a: 2 }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should include files', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $include: x.yaml }',
'x.yaml': '3',
});
const source = FileConfigSource.create({ path: tmp.resolve('a.yaml') });
await expect(readN(source, 1)).resolves.toEqual([
[{ data: { a: 3 }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should include with substitution', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $include: "${MY_FILE}.yaml" } ',
'x.yaml': '4',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
substitutionFunc: async name => (name === 'MY_FILE' ? 'x' : undefined),
});
await expect(readN(source, 1)).resolves.toEqual([
[{ data: { a: 4 }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should substitute in include', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $include: x.yaml }',
'x.yaml': '${MY_VALUE}',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
substitutionFunc: async name => (name === 'MY_VALUE' ? '5' : undefined),
});
await expect(readN(source, 1)).resolves.toEqual([
[{ data: { a: '5' }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should watch included files', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $include: x.yaml }',
'x.yaml': '${MY_VALUE}',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
substitutionFunc: async name => (name === 'MY_VALUE' ? '6' : '7'),
});
setTimeout(() => {
tmp.write('x.yaml', '${MY_OTHER_VALUE}');
}, 10);
await expect(readN(source, 2)).resolves.toEqual([
[{ data: { a: '6' }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
[{ data: { a: '7' }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should watch referenced files', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $file: x.txt }',
'x.txt': '8',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
});
setTimeout(() => {
tmp.write('x.txt', '9');
}, 10);
await expect(readN(source, 2)).resolves.toEqual([
[{ data: { a: '8' }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
[{ data: { a: '9' }, context: 'a.yaml', path: tmp.resolve('a.yaml') }],
]);
});
it('should ignore empty files', async () => {
const tmp = await tmpFiles({
'a.yaml': '',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
});
await expect(readN(source, 1)).resolves.toEqual([[]]);
});
it('should error on file', async () => {
const tmp = await tmpFiles({});
const source = FileConfigSource.create({
path: tmp.resolve('not-found.yaml'),
});
await expect(readN(source, 1)).rejects.toThrow(
`Config file "${tmp.resolve('not-found.yaml')}" does not exist`,
);
});
it('should error on missing include', async () => {
const tmp = await tmpFiles({
'a.yaml': 'a: { $include: not-found.yaml } ',
});
const source = FileConfigSource.create({
path: tmp.resolve('a.yaml'),
});
await expect(readN(source, 1)).rejects.toThrow(
`Failed to read config file at "${tmp.resolve(
'a.yaml',
)}", error at .a, failed to include "${tmp.resolve(
'not-found.yaml',
)}", file does not exist`,
);
});
it('should refuse relative paths', async () => {
expect(() =>
FileConfigSource.create({
path: 'a.yaml',
}),
).toThrow('Config load path is not absolute: "a.yaml"');
});
});
packages/config-loader/src/sources/FileConfigSource.ts
+192
View File
@@ -0,0 +1,192 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import chokidar, { FSWatcher } from 'chokidar';
import fs from 'fs-extra';
import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path';
import yaml from 'yaml';
import {
AsyncConfigSourceIterator,
ConfigSource,
ConfigSourceData,
SubstitutionFunc,
ReadConfigDataOptions,
} from './types';
import { createConfigTransformer } from './transform';
import { NotFoundError } from '@backstage/errors';
/**
* Options for {@link FileConfigSource.create}.
*
* @public
*/
export interface FileConfigSourceOptions {
/**
* The path to the config file that should be loaded.
*/
path: string;
/**
* A substitution function to use instead of the default environment substitution.
*/
substitutionFunc?: SubstitutionFunc;
}
async function readFile(path: string): Promise<string | undefined> {
try {
return await fs.readFile(path, 'utf8');
} catch (error) {
if (error.code === 'ENOENT') {
return undefined;
}
throw error;
}
}
/**
* A config source that loads configuration from a local file.
*
* @public
*/
export class FileConfigSource implements ConfigSource {
/**
* Creates a new config source that loads configuration from the given path.
*
* @remarks
*
* The source will watch the file for changes, as well as any referenced files.
*
* @param options - Options for the config source.
* @returns A new config source that loads from the given path.
*/
static create(options: FileConfigSourceOptions): ConfigSource {
if (!isAbsolute(options.path)) {
throw new Error(`Config load path is not absolute: "${options.path}"`);
}
return new FileConfigSource(options);
}
readonly #path: string;
readonly #substitutionFunc?: SubstitutionFunc;
private constructor(options: FileConfigSourceOptions) {
this.#path = options.path;
this.#substitutionFunc = options.substitutionFunc;
}
// Work is duplicated across each read, in practice that should not
// have any impact since there won't be multiple consumers. If that
// changes it might be worth refactoring this to avoid duplicate work.
async *readConfigData(
options?: ReadConfigDataOptions,
): AsyncConfigSourceIterator {
const signal = options?.signal;
const configFileName = basename(this.#path);
// Keep track of watched paths, since this is simpler than resetting the watcher
const watchedPaths = new Array<string>();
const watcher = chokidar.watch(this.#path, {
usePolling: process.env.NODE_ENV === 'test',
});
const dir = dirname(this.#path);
const transformer = createConfigTransformer({
substitutionFunc: this.#substitutionFunc,
readFile: async path => {
const fullPath = resolvePath(dir, path);
// Any files discovered while reading this config should be watched too
watcher.add(fullPath);
watchedPaths.push(fullPath);
const data = await readFile(fullPath);
if (data === undefined) {
throw new NotFoundError(
`failed to include "${fullPath}", file does not exist`,
);
}
return data;
},
});
// This is the entry point for reading the file, called initially and on change
const readConfigFile = async (): Promise<ConfigSourceData[]> => {
// We clear the watched files every time we initiate a new read
watcher.unwatch(watchedPaths);
watchedPaths.length = 0;
watcher.add(this.#path);
watchedPaths.push(this.#path);
const content = await readFile(this.#path);
if (content === undefined) {
throw new NotFoundError(`Config file "${this.#path}" does not exist`);
}
const parsed = yaml.parse(content);
if (parsed === null) {
return [];
}
try {
const data = await transformer(parsed, { dir });
return [{ data, context: configFileName, path: this.#path }];
} catch (error) {
throw new Error(
`Failed to read config file at "${this.#path}", ${error.message}`,
);
}
};
const onAbort = () => {
signal?.removeEventListener('abort', onAbort);
watcher.close();
};
signal?.addEventListener('abort', onAbort);
yield { configs: await readConfigFile() };
for (;;) {
const event = await this.#waitForEvent(watcher, signal);
if (event === 'abort') {
return;
}
yield { configs: await readConfigFile() };
}
}
toString() {
return `FileConfigSource{path="${this.#path}"}`;
}
#waitForEvent(
watcher: FSWatcher,
signal?: AbortSignal,
): Promise<'change' | 'abort'> {
return new Promise(resolve => {
function onChange() {
resolve('change');
onDone();
}
function onAbort() {
resolve('abort');
onDone();
}
function onDone() {
watcher.removeListener('change', onChange);
signal?.removeEventListener('abort', onAbort);
}
watcher.addListener('change', onChange);
signal?.addEventListener('abort', onAbort);
});
}
}
packages/config-loader/src/sources/MergedConfigSource.test.ts
+151
View File
@@ -0,0 +1,151 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { MergedConfigSource } from './MergedConfigSource';
import { MutableConfigSource } from './MutableConfigSource';
import { isResolved, readAll, simpleSource } from './__testUtils__/testUtils';
import { ConfigSource } from './types';
describe('MergedConfigSource', () => {
it('should forward from a single source', async () => {
const source = simpleSource([{ a: 1 }, { a: 2 }, { a: 3 }]);
const merged = MergedConfigSource.from([source]);
await expect(readAll(merged)).resolves.toEqual([
[{ data: { a: 1 }, context: 'mock-source' }],
[{ data: { a: 2 }, context: 'mock-source' }],
[{ data: { a: 3 }, context: 'mock-source' }],
]);
});
it('should forward from multiple sources', async () => {
const sourceA = simpleSource([{ a: 1 }, { a: 2 }, { a: 3 }], 'a');
const sourceB = simpleSource([{ b: 1 }, { b: 2 }], 'b');
const sourceC = simpleSource([{ c: 1 }], 'c');
const merged = MergedConfigSource.from([sourceA, sourceB, sourceC]);
await expect(readAll(merged)).resolves.toEqual([
[
{ data: { a: 1 }, context: 'a' },
{ data: { b: 1 }, context: 'b' },
{ data: { c: 1 }, context: 'c' },
],
[
{ data: { a: 2 }, context: 'a' },
{ data: { b: 1 }, context: 'b' },
{ data: { c: 1 }, context: 'c' },
],
[
{ data: { a: 3 }, context: 'a' },
{ data: { b: 1 }, context: 'b' },
{ data: { c: 1 }, context: 'c' },
],
[
{ data: { a: 3 }, context: 'a' },
{ data: { b: 2 }, context: 'b' },
{ data: { c: 1 }, context: 'c' },
],
]);
});
it('should forward from multiple sources at difference pace', async () => {
const sourceA = MutableConfigSource.create({ context: 'a' });
const sourceB = MutableConfigSource.create({ context: 'b' });
const merged = MergedConfigSource.from([sourceA, sourceB]);
const it = merged.readConfigData();
const first = it.next();
await expect(isResolved(first, { wait: true })).resolves.toBe(false);
sourceA.setData({ a: 1 });
await expect(isResolved(first, { wait: true })).resolves.toBe(false);
sourceB.setData({ b: 1 });
await expect(first).resolves.toEqual({
value: {
configs: [
{ data: { a: 1 }, context: 'a' },
{ data: { b: 1 }, context: 'b' },
],
},
done: false,
});
sourceB.setData({ b: 2 });
await expect(it.next()).resolves.toEqual({
value: {
configs: [
{ data: { a: 1 }, context: 'a' },
{ data: { b: 2 }, context: 'b' },
],
},
done: false,
});
const last = it.next();
await expect(isResolved(last, { wait: true })).resolves.toBe(false);
sourceA.close();
await expect(isResolved(last, { wait: true })).resolves.toBe(false);
sourceB.close();
await expect(isResolved(last, { wait: true })).resolves.toBe(true);
await expect(last).resolves.toEqual({
done: true,
});
});
it('should be flattened', async () => {
const sym = Symbol.for(
'@backstage/config-loader#MergedConfigSource.sources',
);
const sourceA: ConfigSource = {
async *readConfigData() {
yield { configs: [] };
},
};
const sourceD: ConfigSource = {
async *readConfigData() {
yield { configs: [] };
},
};
const sourceB: ConfigSource = {
async *readConfigData() {
yield { configs: [] };
},
};
const sourceC: ConfigSource = {
async *readConfigData() {
yield { configs: [] };
},
};
const sourceAB = MergedConfigSource.from([sourceA, sourceB]);
const sourceABC = MergedConfigSource.from([sourceAB, sourceC]);
const sourceABCD = MergedConfigSource.from([sourceABC, sourceD]);
expect((sourceAB as any)[sym]).toEqual([sourceA, sourceB]);
expect((sourceABC as any)[sym]).toEqual([sourceA, sourceB, sourceC]);
expect((sourceABCD as any)[sym]).toEqual([
sourceA,
sourceB,
sourceC,
sourceD,
]);
await expect(readAll(sourceAB)).resolves.toEqual([[]]);
await expect(readAll(sourceABC)).resolves.toEqual([[]]);
await expect(readAll(sourceABCD)).resolves.toEqual([[]]);
});
});
packages/config-loader/src/sources/MergedConfigSource.ts
+118
View File
@@ -0,0 +1,118 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
AsyncConfigSourceIterator,
ConfigSource,
ConfigSourceData,
ReadConfigDataOptions,
} from './types';
const sourcesSymbol = Symbol.for(
'@backstage/config-loader#MergedConfigSource.sources',
);
/** @internal */
export class MergedConfigSource implements ConfigSource {
// An optimization to flatten nested merged sources to avid unnecessary microtasks
static #flattenSources(sources: ConfigSource[]): ConfigSource[] {
return sources.flatMap(source => {
if (
sourcesSymbol in source &&
Array.isArray((source as any)[sourcesSymbol])
) {
return this.#flattenSources(
(source as any)[sourcesSymbol] as ConfigSource[],
);
}
return source;
});
}
static from(sources: ConfigSource[]): ConfigSource {
return new MergedConfigSource(this.#flattenSources(sources));
}
[sourcesSymbol]: ConfigSource[];
private constructor(private readonly sources: ConfigSource[]) {
this[sourcesSymbol] = this.sources;
}
async *readConfigData(
options?: ReadConfigDataOptions,
): AsyncConfigSourceIterator {
const its = this.sources.map(source => source.readConfigData(options));
const initialResults = await Promise.all(its.map(it => it.next()));
const configs = initialResults.map((result, i) => {
if (result.done) {
throw new Error(
`Config source ${String(this.sources[i])} returned no data`,
);
}
return result.value.configs;
});
yield { configs: configs.flat(1) };
const results: Array<
| Promise<
readonly [
number,
IteratorResult<{ configs: ConfigSourceData[] }, void>,
]
>
| undefined
> = its.map((it, i) => nextWithIndex(it, i));
while (results.some(Boolean)) {
try {
const [i, result] = (await Promise.race(results.filter(Boolean)))!;
if (result.done) {
results[i] = undefined;
} else {
results[i] = nextWithIndex(its[i], i);
configs[i] = result.value.configs;
yield { configs: configs.flat(1) };
}
} catch (error) {
const source = this.sources[error.index];
if (source) {
throw new Error(`Config source ${String(source)} failed: ${error}`);
}
throw error;
}
}
}
toString() {
return `MergedConfigSource{${this.sources.map(String).join(', ')}}`;
}
}
// Helper to wait for the next value of the iterator, while decorating the value
// or error with the index of the iterator.
function nextWithIndex<T>(
iterator: AsyncIterator<T, void, void>,
index: number,
): Promise<readonly [index: number, result: IteratorResult<T, void>]> {
return iterator.next().then(
r => [index, r] as const,
e => {
throw Object.assign(e, { index });
},
);
}
packages/config-loader/src/sources/MutableConfigSource.test.ts
+125
View File
@@ -0,0 +1,125 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigSources } from './ConfigSources';
import { MutableConfigSource } from './MutableConfigSource';
import { isResolved, readAll } from './__testUtils__/testUtils';
describe('MutableConfigSource', () => {
it('should be initialized with data', async () => {
const source = MutableConfigSource.create({ data: { a: 1 } });
const config = await ConfigSources.toConfig(source);
expect(config.getNumber('a')).toEqual(1);
config.close();
});
it('should be created without data', async () => {
const source = MutableConfigSource.create();
const it = source.readConfigData();
const first = it.next();
await expect(isResolved(first)).resolves.toBe(false);
source.setData({ a: 1 });
await expect(first).resolves.toEqual({
value: {
configs: [
{
data: { a: 1 },
context: 'mutable-config',
},
],
},
done: false,
});
});
it('should be mutable and work with multiple consumers', async () => {
const source = MutableConfigSource.create({ data: { a: 1 } });
const resultsPromise = readAll(source);
const it = source.readConfigData();
await expect(it.next()).resolves.toEqual({
value: {
configs: [
{
data: { a: 1 },
context: 'mutable-config',
},
],
},
done: false,
});
const next2 = it.next();
source.setData({ a: 2 });
await expect(next2).resolves.toEqual({
value: {
configs: [
{
data: { a: 2 },
context: 'mutable-config',
},
],
},
done: false,
});
const next3 = it.next();
source.setData({ a: 3 });
await expect(next3).resolves.toEqual({
value: {
configs: [
{
data: { a: 3 },
context: 'mutable-config',
},
],
},
done: false,
});
const last = it.next();
source.close();
await expect(last).resolves.toEqual({
done: true,
});
await expect(resultsPromise).resolves.toEqual([
[{ data: { a: 1 }, context: 'mutable-config' }],
[{ data: { a: 2 }, context: 'mutable-config' }],
[{ data: { a: 3 }, context: 'mutable-config' }],
]);
});
it('should be self-mutable', async () => {
const source = MutableConfigSource.create({ data: { a: 1 } });
const resultsPromise = readAll(source);
for await (const { configs } of source.readConfigData()) {
const a = configs[0].data.a as number;
if (a < 3) {
source.setData({ a: a + 1 });
} else {
source.close();
}
}
await expect(resultsPromise).resolves.toEqual([
[{ data: { a: 1 }, context: 'mutable-config' }],
[{ data: { a: 2 }, context: 'mutable-config' }],
[{ data: { a: 3 }, context: 'mutable-config' }],
]);
});
});
packages/config-loader/src/sources/MutableConfigSource.ts
+117
View File
@@ -0,0 +1,117 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject } from '@backstage/types';
import {
AsyncConfigSourceIterator,
ConfigSource,
ReadConfigDataOptions,
} from './types';
import { simpleDefer, SimpleDeferred, waitOrAbort } from './utils';
/**
* Options for {@link MutableConfigSource.create}.
*
* @public
*/
export interface MutableConfigSourceOptions {
data?: JsonObject;
context?: string;
}
/**
* A config source that can be updated with new data.
*
* @public
*/
export class MutableConfigSource implements ConfigSource {
/**
* Creates a new mutable config source.
*
* @param options - Options for the config source.
* @returns A new mutable config source.
*/
static create(options?: MutableConfigSourceOptions): MutableConfigSource {
return new MutableConfigSource(
options?.context ?? 'mutable-config',
options?.data,
);
}
#currentData?: JsonObject;
#deferred: SimpleDeferred<void>;
readonly #context: string;
readonly #abortController = new AbortController();
private constructor(context: string, initialData?: JsonObject) {
this.#currentData = initialData;
this.#context = context;
this.#deferred = simpleDefer();
}
async *readConfigData(
options?: ReadConfigDataOptions | undefined,
): AsyncConfigSourceIterator {
let deferredPromise = this.#deferred.promise;
if (this.#currentData !== undefined) {
yield { configs: [{ data: this.#currentData, context: this.#context }] };
}
for (;;) {
const [ok] = await waitOrAbort(deferredPromise, [
options?.signal,
this.#abortController.signal,
]);
if (!ok) {
return;
}
deferredPromise = this.#deferred.promise;
if (this.#currentData !== undefined) {
yield {
configs: [{ data: this.#currentData, context: this.#context }],
};
}
}
}
/**
* Set the data of the config source.
*
* @param data - The new data to set
*/
setData(data: JsonObject): void {
if (!this.#abortController.signal.aborted) {
this.#currentData = data;
const oldDeferred = this.#deferred;
this.#deferred = simpleDefer();
oldDeferred.resolve();
}
}
/**
* Close the config source, preventing any further updates.
*/
close(): void {
this.#currentData = undefined;
this.#abortController.abort();
}
toString() {
return `MutableConfigSource{}`;
}
}
packages/config-loader/src/sources/ObservableConfigProxy.test.ts
+143
View File
@@ -0,0 +1,143 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { ObservableConfigProxy } from './ObservableConfigProxy';
describe('ObservableConfigProxy', () => {
it('should notify subscribers', () => {
const config = ObservableConfigProxy.create(new AbortController());
const fn = jest.fn();
const sub = config.subscribe(fn);
expect(config.getOptionalNumber('x')).toBe(undefined);
config.setConfig(new ConfigReader({}));
expect(fn).toHaveBeenCalledTimes(1);
expect(config.getOptionalNumber('x')).toBe(undefined);
config.setConfig(new ConfigReader({ x: 1 }));
expect(fn).toHaveBeenCalledTimes(2);
expect(config.getOptionalNumber('x')).toBe(1);
config.setConfig(new ConfigReader({ x: 3 }));
expect(fn).toHaveBeenCalledTimes(3);
sub.unsubscribe();
expect(config.getOptionalNumber('x')).toBe(3);
config.setConfig(new ConfigReader({ x: 5 }));
expect(fn).toHaveBeenCalledTimes(3);
expect(config.getOptionalNumber('x')).toBe(5);
});
it('should forward subscriptions', () => {
const config1 = ObservableConfigProxy.create(new AbortController());
const fn1 = jest.fn();
const fn2 = jest.fn();
const fn3 = jest.fn();
const config2 = config1.getConfig('a');
const config3 = config2.getConfig('b');
const sub1 = config1.subscribe(fn1);
const sub2 = config2.subscribe!(fn2);
const sub3 = config3.subscribe!(fn3);
expect(config1.getOptionalNumber('x')).toBe(undefined);
expect(config2.getOptionalNumber('x')).toBe(undefined);
expect(config3.getOptionalNumber('x')).toBe(undefined);
config1.setConfig(new ConfigReader({}));
expect(fn1).toHaveBeenCalledTimes(1);
expect(fn2).toHaveBeenCalledTimes(1);
expect(fn3).toHaveBeenCalledTimes(1);
expect(config1.getOptionalNumber('x')).toBe(undefined);
expect(config2.getOptionalNumber('x')).toBe(undefined);
expect(config3.getOptionalNumber('x')).toBe(undefined);
config1.setConfig(new ConfigReader({ x: 1, a: { x: 2, b: { x: 3 } } }));
expect(fn1).toHaveBeenCalledTimes(2);
expect(fn2).toHaveBeenCalledTimes(2);
expect(fn3).toHaveBeenCalledTimes(2);
expect(config1.getNumber('x')).toBe(1);
expect(config2.getNumber('x')).toBe(2);
expect(config3.getNumber('x')).toBe(3);
sub1.unsubscribe();
sub2.unsubscribe();
sub3.unsubscribe();
config1.setConfig(new ConfigReader({ x: 4, a: { x: 5, b: { x: 6 } } }));
expect(fn1).toHaveBeenCalledTimes(2);
expect(fn2).toHaveBeenCalledTimes(2);
expect(fn3).toHaveBeenCalledTimes(2);
expect(config1.getNumber('x')).toBe(4);
expect(config2.getNumber('x')).toBe(5);
expect(config3.getNumber('x')).toBe(6);
config1.setConfig(new ConfigReader({}));
expect(() => config1.getNumber('x')).toThrow(
"Missing required config value at 'x'",
);
expect(() => config2.getNumber('x')).toThrow(
"Missing required config value at 'a'",
);
expect(() => config3.getNumber('x')).toThrow(
"Missing required config value at 'a'",
);
config1.setConfig(
new ConfigReader({ x: 's', a: { x: 's', b: { x: 's' } } }),
);
expect(() => config1.getNumber('x')).toThrow(
"Unable to convert config value for key 'x' in 'mock-config' to a number",
);
expect(() => config2.getNumber('x')).toThrow(
"Unable to convert config value for key 'a.x' in 'mock-config' to a number",
);
expect(() => config3.getNumber('x')).toThrow(
"Unable to convert config value for key 'a.b.x' in 'mock-config' to a number",
);
});
it('should make sub configs available as expected', () => {
const config = ObservableConfigProxy.create(new AbortController());
config.setConfig(new ConfigReader({ a: { x: 1 } }));
expect(config.getConfig('a')).toBeDefined();
expect(config.getConfig('a').getNumber('x')).toBe(1);
expect(config.getConfig('a').getOptionalNumber('x')).toBe(1);
expect(config.getOptionalConfig('a')?.getNumber('x')).toBe(1);
expect(config.getOptionalConfig('a')?.getOptionalNumber('x')).toBe(1);
expect(config.getOptionalConfig('b')).toBeUndefined();
expect(() => config.getConfig('b')).toBeDefined();
expect(() => config.getConfig('b').get()).toThrow();
});
it('should be closed', () => {
const controller = new AbortController();
const config = ObservableConfigProxy.create(controller);
config.setConfig(new ConfigReader({ a: { x: 1 } }));
expect(controller.signal.aborted).toBe(false);
config.close();
expect(controller.signal.aborted).toBe(true);
config.close();
expect(() => (config.getConfig('a') as any).close()).toThrow(
'Only the root config can be closed',
);
});
});
packages/config-loader/src/sources/ObservableConfigProxy.ts
+140
View File
@@ -0,0 +1,140 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config, ConfigReader } from '@backstage/config';
import { JsonValue } from '@backstage/types';
export class ObservableConfigProxy implements Config {
private config: Config = new ConfigReader({});
private readonly subscribers: (() => void)[] = [];
static create(abortController: AbortController): ObservableConfigProxy {
return new ObservableConfigProxy(undefined, undefined, abortController);
}
private constructor(
private readonly parent?: ObservableConfigProxy,
private readonly parentKey?: string,
private readonly abortController?: AbortController,
) {
if (parent && !parentKey) {
throw new Error('parentKey is required if parent is set');
}
}
setConfig(config: Config) {
if (this.parent) {
throw new Error('immutable');
}
this.config = config;
for (const subscriber of this.subscribers) {
try {
subscriber();
} catch (error) {
console.error(`Config subscriber threw error, ${error}`);
}
}
}
close() {
if (!this.abortController) {
throw new Error('Only the root config can be closed');
}
this.abortController.abort();
}
subscribe(onChange: () => void): { unsubscribe: () => void } {
if (this.parent) {
return this.parent.subscribe(onChange);
}
this.subscribers.push(onChange);
return {
unsubscribe: () => {
const index = this.subscribers.indexOf(onChange);
if (index >= 0) {
this.subscribers.splice(index, 1);
}
},
};
}
private select(required: true): Config;
private select(required: false): Config | undefined;
private select(required: boolean): Config | undefined {
if (this.parent && this.parentKey) {
if (required) {
return this.parent.select(true).getConfig(this.parentKey);
}
return this.parent.select(false)?.getOptionalConfig(this.parentKey);
}
return this.config;
}
has(key: string): boolean {
return this.select(false)?.has(key) ?? false;
}
keys(): string[] {
return this.select(false)?.keys() ?? [];
}
get<T = JsonValue>(key?: string): T {
return this.select(true).get(key);
}
getOptional<T = JsonValue>(key?: string): T | undefined {
return this.select(false)?.getOptional(key);
}
getConfig(key: string): Config {
return new ObservableConfigProxy(this, key);
}
getOptionalConfig(key: string): Config | undefined {
if (this.select(false)?.has(key)) {
return new ObservableConfigProxy(this, key);
}
return undefined;
}
getConfigArray(key: string): Config[] {
return this.select(true).getConfigArray(key);
}
getOptionalConfigArray(key: string): Config[] | undefined {
return this.select(false)?.getOptionalConfigArray(key);
}
getNumber(key: string): number {
return this.select(true).getNumber(key);
}
getOptionalNumber(key: string): number | undefined {
return this.select(false)?.getOptionalNumber(key);
}
getBoolean(key: string): boolean {
return this.select(true).getBoolean(key);
}
getOptionalBoolean(key: string): boolean | undefined {
return this.select(false)?.getOptionalBoolean(key);
}
getString(key: string): string {
return this.select(true).getString(key);
}
getOptionalString(key: string): string | undefined {
return this.select(false)?.getOptionalString(key);
}
getStringArray(key: string): string[] {
return this.select(true).getStringArray(key);
}
getOptionalStringArray(key: string): string[] | undefined {
return this.select(false)?.getOptionalStringArray(key);
}
}
packages/config-loader/src/sources/RemoteConfigSource.test.ts
+85
View File
@@ -0,0 +1,85 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { rest } from 'msw';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { setupServer } from 'msw/node';
import { RemoteConfigSource } from './RemoteConfigSource';
import { readN } from './__testUtils__/testUtils';
describe('RemoteConfigSource', () => {
const worker = setupServer();
setupRequestMockHandlers(worker);
it('should load config from a remote URL', async () => {
worker.use(
rest.get('http://localhost/config.yaml', (_req, res, ctx) =>
res(
ctx.body(`
app:
title: Example App
substituted: \${VALUE}
escaped: \$\${VALUE}
`),
),
),
);
const source = RemoteConfigSource.create({
url: 'http://localhost/config.yaml',
substitutionFunc: async () => 'x',
});
await expect(readN(source, 1)).resolves.toEqual([
[
{
context: 'http://localhost/config.yaml',
data: {
app: {
title: 'Example App',
substituted: 'x',
escaped: '${VALUE}',
},
},
},
],
]);
});
it('should reload config from a remote URL', async () => {
let fetched = false;
worker.use(
rest.get('http://localhost/config.yaml', (_req, res, ctx) => {
if (!fetched) {
fetched = true;
return res(ctx.body('x: 1'));
}
return res(ctx.body('x: 2'));
}),
);
const source = RemoteConfigSource.create({
url: 'http://localhost/config.yaml',
reloadInterval: { seconds: 0 },
});
await expect(readN(source, 2)).resolves.toEqual([
[{ context: 'http://localhost/config.yaml', data: { x: 1 } }],
[{ context: 'http://localhost/config.yaml', data: { x: 2 } }],
]);
});
});
packages/config-loader/src/sources/RemoteConfigSource.ts
+181
View File
@@ -0,0 +1,181 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ResponseError } from '@backstage/errors';
import { HumanDuration, JsonObject } from '@backstage/types';
import isEqual from 'lodash/isEqual';
import fetch from 'node-fetch';
import yaml from 'yaml';
import { ConfigTransformer, createConfigTransformer } from './transform';
import {
AsyncConfigSourceIterator,
ConfigSource,
SubstitutionFunc,
ReadConfigDataOptions,
} from './types';
const DEFAULT_RELOAD_INTERVAL = { seconds: 60 };
function durationToMs(duration: HumanDuration): number {
const {
years = 0,
months = 0,
weeks = 0,
days = 0,
hours = 0,
minutes = 0,
seconds = 0,
milliseconds = 0,
} = duration;
const totalDays = years * 365 + months * 30 + weeks * 7 + days;
const totalHours = totalDays * 24 + hours;
const totalMinutes = totalHours * 60 + minutes;
const totalSeconds = totalMinutes * 60 + seconds;
const totalMilliseconds = totalSeconds * 1000 + milliseconds;
return totalMilliseconds;
}
/**
* Options for {@link RemoteConfigSource.create}.
*
* @public
*/
export interface RemoteConfigSourceOptions {
/**
* The URL to load the config from.
*/
url: string;
/**
* How often to reload the config from the remote URL, defaults to 1 minute.
*
* Set to Infinity to disable reloading, for example `{ days: Infinity }`.
*/
reloadInterval?: HumanDuration;
/**
* A substitution function to use instead of the default environment substitution.
*/
substitutionFunc?: SubstitutionFunc;
}
/**
* A config source that loads configuration from a remote URL.
*
* @public
*/
export class RemoteConfigSource implements ConfigSource {
/**
* Creates a new {@link RemoteConfigSource}.
*
* @param options - Options for the source.
* @returns A new remote config source.
*/
static create(options: RemoteConfigSourceOptions): ConfigSource {
try {
// eslint-disable-next-line no-new
new URL(options.url);
} catch (error) {
throw new Error(
`Invalid URL provided to remote config source, '${options.url}', ${error}`,
);
}
return new RemoteConfigSource(options);
}
readonly #url: string;
readonly #reloadIntervalMs: number;
readonly #transformer: ConfigTransformer;
private constructor(options: RemoteConfigSourceOptions) {
this.#url = options.url;
this.#reloadIntervalMs = durationToMs(
options.reloadInterval ?? DEFAULT_RELOAD_INTERVAL,
);
this.#transformer = createConfigTransformer({
substitutionFunc: options.substitutionFunc,
});
}
async *readConfigData(
options?: ReadConfigDataOptions | undefined,
): AsyncConfigSourceIterator {
let data = await this.#load();
yield { configs: [{ data, context: this.#url }] };
for (;;) {
await this.#wait(options?.signal);
if (options?.signal?.aborted) {
return;
}
try {
const newData = await this.#load(options?.signal);
if (newData && !isEqual(data, newData)) {
data = newData;
yield { configs: [{ data, context: this.#url }] };
}
} catch (error) {
if (error.name !== 'AbortError') {
console.error(`Failed to read config from ${this.#url}, ${error}`);
}
}
}
}
toString() {
return `RemoteConfigSource{path="${this.#url}"}`;
}
async #load(signal?: AbortSignal): Promise<JsonObject> {
const res = await fetch(this.#url, {
signal: signal as import('node-fetch').RequestInit['signal'],
});
if (!res.ok) {
throw ResponseError.fromResponse(res);
}
const content = await res.text();
const data = await this.#transformer(yaml.parse(content));
if (data === null) {
throw new Error('configuration data is null');
} else if (typeof data !== 'object') {
throw new Error('configuration data is not an object');
} else if (Array.isArray(data)) {
throw new Error(
'configuration data is an array, expected an object instead',
);
}
return data;
}
async #wait(signal?: AbortSignal) {
return new Promise<void>(resolve => {
const timeoutId = setTimeout(onDone, this.#reloadIntervalMs);
signal?.addEventListener('abort', onDone);
function onDone() {
clearTimeout(timeoutId);
signal?.removeEventListener('abort', onDone);
resolve();
}
});
}
}
packages/config-loader/src/sources/StaticConfigSource.test.ts
+49
View File
@@ -0,0 +1,49 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject } from '@backstage/types';
import { StaticConfigSource } from './StaticConfigSource';
import { readAll } from './__testUtils__/testUtils';
import ZenObservable from 'zen-observable';
describe('StaticConfigSource', () => {
it('should be created from data', async () => {
const source = StaticConfigSource.create({ data: { a: 1 } });
await expect(readAll(source)).resolves.toEqual([
[{ data: { a: 1 }, context: 'static-config' }],
]);
});
it('should be created from promise', async () => {
const source = StaticConfigSource.create({
data: Promise.resolve({ a: 1 }),
});
await expect(readAll(source)).resolves.toEqual([
[{ data: { a: 1 }, context: 'static-config' }],
]);
});
it('should be created from observable', async () => {
const source = StaticConfigSource.create({
data: ZenObservable.of<JsonObject>({ a: 1 }, { a: 2 }, { a: 3 }),
});
await expect(readAll(source)).resolves.toEqual([
[{ data: { a: 1 }, context: 'static-config' }],
[{ data: { a: 2 }, context: 'static-config' }],
[{ data: { a: 3 }, context: 'static-config' }],
]);
});
});
packages/config-loader/src/sources/StaticConfigSource.ts
+148
View File
@@ -0,0 +1,148 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject, Observable } from '@backstage/types';
import {
AsyncConfigSourceIterator,
ConfigSource,
ReadConfigDataOptions,
} from './types';
import { simpleDefer } from './utils';
/**
* Options for {@link StaticConfigSource.create}.
*
* @public
*/
export interface StaticConfigSourceOptions {
data:
| JsonObject
| Observable<JsonObject>
| PromiseLike<JsonObject>
| AsyncIterable<JsonObject>;
context?: string;
}
/** @internal */
class StaticObservableConfigSource implements ConfigSource {
constructor(
private readonly data: Observable<JsonObject>,
private readonly context: string,
) {}
async *readConfigData(
options?: ReadConfigDataOptions | undefined,
): AsyncConfigSourceIterator {
const queue = new Array<JsonObject>();
let deferred = simpleDefer<void>();
const sub = this.data.subscribe({
next(value) {
queue.push(value);
deferred.resolve();
deferred = simpleDefer();
},
complete() {
deferred.resolve();
},
});
const signal = options?.signal;
if (signal) {
const onAbort = () => {
sub.unsubscribe();
queue.length = 0;
deferred.resolve();
signal.removeEventListener('abort', onAbort);
};
signal.addEventListener('abort', onAbort);
}
for (;;) {
await deferred.promise;
if (queue.length === 0) {
return;
}
while (queue.length > 0) {
yield { configs: [{ data: queue.shift()!, context: this.context }] };
}
}
}
}
function isObservable<T>(value: {}): value is Observable<T> {
return 'subscribe' in value && typeof (value as any).subscribe === 'function';
}
function isAsyncIterable<T>(value: {}): value is AsyncIterable<T> {
return Symbol.asyncIterator in value;
}
/**
* A configuration source that reads from a static object, promise, iterable, or observable.
*
* @public
*/
export class StaticConfigSource implements ConfigSource {
/**
* Creates a new {@link StaticConfigSource}.
*
* @param options - Options for the config source
* @returns A new static config source
*/
static create(options: StaticConfigSourceOptions): ConfigSource {
const { data, context = 'static-config' } = options;
if (!data) {
return {
async *readConfigData(): AsyncConfigSourceIterator {
yield { configs: [] };
return;
},
};
}
if (isObservable<JsonObject>(data)) {
return new StaticObservableConfigSource(data, context);
}
if (isAsyncIterable(data)) {
return {
async *readConfigData(): AsyncConfigSourceIterator {
for await (const value of data) {
yield { configs: [{ data: value, context }] };
}
},
};
}
return new StaticConfigSource(data, context);
}
private constructor(
private readonly promise: JsonObject | PromiseLike<JsonObject>,
private readonly context: string,
) {}
async *readConfigData(): AsyncConfigSourceIterator {
yield { configs: [{ data: await this.promise, context: this.context }] };
return;
}
toString() {
return `StaticConfigSource{}`;
}
}
packages/config-loader/src/sources/__fixtures__/a.yaml
+1
View File
@@ -0,0 +1 @@
a: 1
packages/config-loader/src/sources/__testUtils__/testUtils.ts
+89
View File
@@ -0,0 +1,89 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject } from '@backstage/types';
import { ConfigSource, ConfigSourceData } from '../types';
export function isResolved(
promise: Promise<unknown>,
{ wait }: { wait?: number | boolean } = {},
): Promise<boolean> {
return Promise.race([
promise.then(() => true),
typeof wait !== 'undefined'
? new Promise<boolean>(resolve =>
setTimeout(
() => resolve(false),
typeof wait === 'number' ? wait : 10,
),
)
: Promise.resolve().then(() => false),
]);
}
export async function readAll(
source: ConfigSource,
signal?: AbortSignal,
): Promise<ConfigSourceData[][]> {
const results: ConfigSourceData[][] = [];
try {
for await (const { configs } of source.readConfigData({ signal })) {
results.push(configs);
}
} catch (error) {
throw error;
}
return results;
}
export async function readN(
source: ConfigSource,
n: number,
): Promise<ConfigSourceData[][]> {
const results: ConfigSourceData[][] = [];
try {
const controller = new AbortController();
for await (const { configs } of source.readConfigData({
signal: controller.signal,
})) {
results.push(configs);
if (results.length >= n) {
break;
}
}
controller.abort();
} catch (error) {
throw error;
}
return results;
}
export function simpleSource(
data: JsonObject[],
context: string = 'mock-source',
): ConfigSource {
return {
async *readConfigData() {
for (const d of data) {
yield { configs: [{ data: d, context }] };
}
},
};
}
packages/config-loader/src/sources/index.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { ConfigSources } from './ConfigSources';
export type {
BaseConfigSourcesOptions,
ClosableConfig,
ConfigSourceTarget,
ConfigSourcesDefaultForTargetsOptions,
ConfigSourcesDefaultOptions,
} from './ConfigSources';
export { EnvConfigSource, readEnvConfig } from './EnvConfigSource';
export type { EnvConfigSourceOptions } from './EnvConfigSource';
export { FileConfigSource } from './FileConfigSource';
export type { FileConfigSourceOptions } from './FileConfigSource';
export { MutableConfigSource } from './MutableConfigSource';
export type { MutableConfigSourceOptions } from './MutableConfigSource';
export { RemoteConfigSource } from './RemoteConfigSource';
export type { RemoteConfigSourceOptions } from './RemoteConfigSource';
export { StaticConfigSource } from './StaticConfigSource';
export type { StaticConfigSourceOptions } from './StaticConfigSource';
export type {
SubstitutionFunc as EnvFunc,
ConfigSource,
ConfigSourceData,
ReadConfigDataOptions,
AsyncConfigSourceIterator,
} from './types';
packages/config-loader/src/lib/transform/apply.test.ts → packages/config-loader/src/sources/transform/apply.test.ts
+3 -3
View File
@@ -19,7 +19,6 @@ import { applyConfigTransforms } from './apply';
describe('applyConfigTransforms', () => {
it('should apply not transforms to input', async () => {
const data = applyConfigTransforms(
'',
{
app: {
title: 'Test',
@@ -28,6 +27,7 @@ describe('applyConfigTransforms', () => {
z: null,
},
},
{},
[],
);
@@ -41,14 +41,13 @@ describe('applyConfigTransforms', () => {
});
it('should throw if input is not an object', async () => {
const config = applyConfigTransforms('', 'not-config', []);
const config = applyConfigTransforms('not-config', {}, []);
await expect(config).rejects.toThrow('expected object at config root');
});
it('should apply transforms', async () => {
const config = applyConfigTransforms(
'',
{
app: {
title: 'Test',
@@ -57,6 +56,7 @@ describe('applyConfigTransforms', () => {
z: null,
},
},
{},
[
async value => {
if (typeof value === 'number') {
packages/config-loader/src/lib/transform/apply.ts → packages/config-loader/src/sources/transform/apply.ts
+37 -6
View File
@@ -16,34 +16,37 @@
import { JsonObject, JsonValue } from '@backstage/types';
import { assertError } from '@backstage/errors';
import { TransformFunc } from './types';
import { TransformContext, TransformFunc } from './types';
import { isObject } from './utils';
import { createSubstitutionTransform } from './substitution';
import { createIncludeTransform } from './include';
import { SubstitutionFunc } from '../types';
/**
* Applies a set of transforms to raw configuration data.
*/
export async function applyConfigTransforms(
initialDir: string,
input: JsonValue,
context: { dir?: string },
transforms: TransformFunc[],
): Promise<JsonObject> {
async function transform(
inputObj: JsonValue,
path: string,
baseDir: string,
baseDir?: string,
): Promise<JsonValue | undefined> {
let obj = inputObj;
let dir = baseDir;
for (const tf of transforms) {
try {
const result = await tf(inputObj, baseDir);
const result = await tf(inputObj, { dir });
if (result.applied) {
if (result.value === undefined) {
return undefined;
}
obj = result.value;
dir = result.newBaseDir ?? dir;
dir = result?.newDir ?? dir;
break;
}
} catch (error) {
@@ -84,9 +87,37 @@ export async function applyConfigTransforms(
return out;
}
const finalData = await transform(input, '', initialDir);
const finalData = await transform(input, '', context?.dir);
if (!isObject(finalData)) {
throw new TypeError('expected object at config root');
}
return finalData;
}
/** @internal */
export type ConfigTransformer = (
input: JsonObject,
context?: TransformContext,
) => Promise<JsonObject>;
/** @internal */
export function createConfigTransformer(options: {
substitutionFunc?: SubstitutionFunc;
readFile?(path: string): Promise<string>;
}): ConfigTransformer {
const { substitutionFunc = async name => process.env[name], readFile } =
options;
const substitutionTransform = createSubstitutionTransform(substitutionFunc);
const transforms = [substitutionTransform];
if (readFile) {
const includeTransform = createIncludeTransform(
substitutionFunc,
readFile,
substitutionTransform,
);
transforms.push(includeTransform);
}
return async (input, ctx) =>
applyConfigTransforms(input, ctx ?? {}, transforms);
}
packages/config-loader/src/lib/transform/include.test.ts → packages/config-loader/src/sources/transform/include.test.ts
+36 -22
View File
@@ -69,34 +69,34 @@ const includeTransform = createIncludeTransform(env, readFile, substitute);
describe('includeTransform', () => {
it('should not transform unknown values', async () => {
await expect(includeTransform('foo', root)).resolves.toEqual({
await expect(includeTransform('foo', { dir: root })).resolves.toEqual({
applied: false,
});
await expect(includeTransform([1], root)).resolves.toEqual({
await expect(includeTransform([1], { dir: root })).resolves.toEqual({
applied: false,
});
await expect(includeTransform(1, root)).resolves.toEqual({
await expect(includeTransform(1, { dir: root })).resolves.toEqual({
applied: false,
});
await expect(includeTransform({ x: 'y' }, root)).resolves.toEqual({
await expect(includeTransform({ x: 'y' }, { dir: root })).resolves.toEqual({
applied: false,
});
await expect(includeTransform(null, root)).resolves.toEqual({
await expect(includeTransform(null, { dir: root })).resolves.toEqual({
applied: false,
});
});
it('should include text files', async () => {
await expect(
includeTransform({ $file: 'my-secret' }, root),
includeTransform({ $file: 'my-secret' }, { dir: root }),
).resolves.toEqual({ applied: true, value: 'secret' });
await expect(
includeTransform({ $file: 'no-secret' }, root),
includeTransform({ $file: 'no-secret' }, { dir: root }),
).rejects.toThrow('File not found!');
});
it('should trim newlines from end of file', async () => {
await expect(
includeTransform({ $file: 'with-newline-at-the-end' }, root),
includeTransform({ $file: 'with-newline-at-the-end' }, { dir: root }),
).resolves.toEqual({
applied: true,
value: 'value without newline at the end',
@@ -104,12 +104,14 @@ describe('includeTransform', () => {
});
it('should include env vars', async () => {
await expect(includeTransform({ $env: 'SECRET' }, root)).resolves.toEqual({
await expect(
includeTransform({ $env: 'SECRET' }, { dir: root }),
).resolves.toEqual({
applied: true,
value: 'my-secret',
});
await expect(
includeTransform({ $env: 'NO_SECRET' }, root),
includeTransform({ $env: 'NO_SECRET' }, { dir: root }),
).resolves.toEqual({
applied: true,
value: undefined,
@@ -119,16 +121,19 @@ describe('includeTransform', () => {
it('should include config files', async () => {
// New format with path in fragment
await expect(
includeTransform({ $include: 'my-data.json#a.b.c' }, root),
includeTransform({ $include: 'my-data.json#a.b.c' }, { dir: root }),
).resolves.toEqual({ applied: true, value: 42 });
await expect(
includeTransform({ $include: 'my-data.json#a.b' }, root),
includeTransform({ $include: 'my-data.json#a.b' }, { dir: root }),
).resolves.toEqual({ applied: true, value: { c: 42 } });
await expect(
includeTransform({ $include: 'my-data.yaml#some.yaml.key' }, root),
includeTransform(
{ $include: 'my-data.yaml#some.yaml.key' },
{ dir: root },
),
).resolves.toEqual({ applied: true, value: 7 });
await expect(
includeTransform({ $include: 'my-data.yaml' }, root),
includeTransform({ $include: 'my-data.yaml' }, { dir: root }),
).resolves.toEqual({
applied: true,
value: {
@@ -136,7 +141,7 @@ describe('includeTransform', () => {
},
});
await expect(
includeTransform({ $include: 'my-data.yaml#' }, root),
includeTransform({ $include: 'my-data.yaml#' }, { dir: root }),
).resolves.toEqual({
applied: true,
value: {
@@ -144,26 +149,32 @@ describe('includeTransform', () => {
},
});
await expect(
includeTransform({ $include: 'my-data.yml#different.key' }, root),
includeTransform(
{ $include: 'my-data.yml#different.key' },
{ dir: root },
),
).resolves.toEqual({ applied: true, value: 'hello' });
});
it('should reject invalid includes', async () => {
await expect(
includeTransform({ $include: 'no-parser.js' }, root),
includeTransform({ $include: 'no-parser.js' }, { dir: root }),
).rejects.toThrow(
'no configuration parser available for included file no-parser.js',
);
await expect(
includeTransform({ $include: 'no-data.yml#different.key' }, root),
includeTransform(
{ $include: 'no-data.yml#different.key' },
{ dir: root },
),
).rejects.toThrow('File not found!');
await expect(
includeTransform({ $include: 'my-data.yml#missing.key' }, root),
includeTransform({ $include: 'my-data.yml#missing.key' }, { dir: root }),
).rejects.toThrow(
"value at 'missing' in included file my-data.yml is not an object",
);
await expect(
includeTransform({ $include: 'invalid.yaml' }, root),
includeTransform({ $include: 'invalid.yaml' }, { dir: root }),
).rejects.toThrow(
/failed to parse included file invalid.yaml, YAMLParseError: Flow sequence in block collection must be sufficiently indented and end with a \] at line 1, column 7:\s+foo: \[\}/,
);
@@ -171,11 +182,14 @@ describe('includeTransform', () => {
it('should call substitute prior to handling includes directive', async () => {
await expect(
includeTransform({ $include: `${substituteMe}/my-data.json` }, root),
includeTransform(
{ $include: `${substituteMe}/my-data.json` },
{ dir: root },
),
).resolves.toEqual({
applied: true,
value: { foo: 'bar' },
newBaseDir: resolvePath(root, mySubstitution),
newDir: resolvePath(root, mySubstitution),
});
});
});
packages/config-loader/src/lib/transform/include.ts → packages/config-loader/src/sources/transform/include.ts
+20 -8
View File
@@ -18,7 +18,8 @@ import yaml from 'yaml';
import { extname, dirname, resolve as resolvePath } from 'path';
import { JsonObject, JsonValue } from '@backstage/types';
import { isObject } from './utils';
import { TransformFunc, EnvFunc, ReadFileFunc } from './types';
import { TransformFunc, ReadFileFunc } from './types';
import { SubstitutionFunc } from '../types';
// Parsers for each type of included file
const includeFileParser: {
@@ -33,11 +34,15 @@ const includeFileParser: {
* Transforms a include description into the actual included value.
*/
export function createIncludeTransform(
env: EnvFunc,
env: SubstitutionFunc,
readFile: ReadFileFunc,
substitute: TransformFunc,
): TransformFunc {
return async (input: JsonValue, baseDir: string) => {
return async (input, context) => {
const { dir } = context;
if (!dir) {
throw new Error('Include transform requires a base directory');
}
if (!isObject(input)) {
return { applied: false };
}
@@ -59,7 +64,7 @@ export function createIncludeTransform(
throw new Error(`${includeKey} include value is not a string`);
}
const substituteResults = await substitute(rawIncludedValue, baseDir);
const substituteResults = await substitute(rawIncludedValue, { dir });
const includeValue = substituteResults.applied
? substituteResults.value
: rawIncludedValue;
@@ -72,7 +77,7 @@ export function createIncludeTransform(
switch (includeKey) {
case '$file':
try {
const value = await readFile(resolvePath(baseDir, includeValue));
const value = await readFile(resolvePath(dir, includeValue));
return { applied: true, value: value.trimEnd() };
} catch (error) {
throw new Error(`failed to read file ${includeValue}, ${error}`);
@@ -95,9 +100,9 @@ export function createIncludeTransform(
);
}
const path = resolvePath(baseDir, filePath);
const path = resolvePath(dir, filePath);
const content = await readFile(path);
const newBaseDir = dirname(path);
const newDir = dirname(path);
const parts = dataPath ? dataPath.split('.') : [];
@@ -121,10 +126,17 @@ export function createIncludeTransform(
value = value[part];
}
if (typeof value === 'string') {
const substituted = await substitute(value, { dir: newDir });
if (substituted.applied) {
value = substituted.value;
}
}
return {
applied: true,
value,
newBaseDir: newBaseDir !== baseDir ? newBaseDir : undefined,
newDir: newDir !== dir ? newDir : undefined,
};
}
packages/config-loader/src/lib/transform/index.ts → packages/config-loader/src/sources/transform/index.ts
+3 -3
View File
@@ -14,6 +14,6 @@
* limitations under the License.
*/
export { applyConfigTransforms } from './apply';
export { createIncludeTransform } from './include';
export { createSubstitutionTransform } from './substitution';
export { createConfigTransformer } from './apply';
export type { ConfigTransformer } from './apply';
export type { ReadFileFunc, TransformContext, TransformFunc } from './types';
packages/config-loader/src/lib/transform/substitution.test.ts → packages/config-loader/src/sources/transform/substitution.test.ts
+24 -16
View File
@@ -29,52 +29,60 @@ const substituteTransform = createSubstitutionTransform(env);
describe('substituteTransform', () => {
it('should not transform unknown values', async () => {
await expect(substituteTransform(false, '/')).resolves.toEqual({
await expect(substituteTransform(false, { dir: '/' })).resolves.toEqual({
applied: false,
});
await expect(substituteTransform([1], '/')).resolves.toEqual({
await expect(substituteTransform([1], { dir: '/' })).resolves.toEqual({
applied: false,
});
await expect(substituteTransform(1, '/')).resolves.toEqual({
await expect(substituteTransform(1, { dir: '/' })).resolves.toEqual({
applied: false,
});
await expect(substituteTransform({ x: 'y' }, '/')).resolves.toEqual({
await expect(
substituteTransform({ x: 'y' }, { dir: '/' }),
).resolves.toEqual({
applied: false,
});
await expect(substituteTransform(null, '/')).resolves.toEqual({
await expect(substituteTransform(null, { dir: '/' })).resolves.toEqual({
applied: false,
});
});
it('should substitute env var', async () => {
await expect(substituteTransform('hello ${SECRET}', '/')).resolves.toEqual({
await expect(
substituteTransform('hello ${SECRET}', { dir: '/' }),
).resolves.toEqual({
applied: true,
value: 'hello my-secret',
});
await expect(
substituteTransform('${SECRET } $${} ${TOKEN }', '/'),
substituteTransform('${SECRET } $${} ${TOKEN }', { dir: '/' }),
).resolves.toEqual({ applied: true, value: 'my-secret ${} my-token' });
await expect(substituteTransform('foo ${MISSING}', '/')).resolves.toEqual({
applied: true,
value: undefined,
});
await expect(
substituteTransform('empty substitute ${}', '/'),
substituteTransform('foo ${MISSING}', { dir: '/' }),
).resolves.toEqual({
applied: true,
value: undefined,
});
await expect(
substituteTransform('foo ${MISSING} ${SECRET}', '/'),
substituteTransform('empty substitute ${}', { dir: '/' }),
).resolves.toEqual({
applied: true,
value: undefined,
});
await expect(
substituteTransform('foo ${MISSING} ${SECRET}', { dir: '/' }),
).resolves.toEqual({ applied: true, value: undefined });
await expect(
substituteTransform('foo ${SECRET} ${SECRET}', '/'),
substituteTransform('foo ${SECRET} ${SECRET}', { dir: '/' }),
).resolves.toEqual({ applied: true, value: 'foo my-secret my-secret' });
await expect(
substituteTransform('foo ${SECRET} $$${ESCAPE_ME}', '/'),
substituteTransform('foo ${SECRET} $$${ESCAPE_ME}', { dir: '/' }),
).resolves.toEqual({ applied: true, value: 'foo my-secret $${ESCAPE_ME}' });
await expect(
substituteTransform('foo $${ESCAPE_ME} $$${ESCAPE_ME_TOO} $${}', '/'),
substituteTransform('foo $${ESCAPE_ME} $$${ESCAPE_ME_TOO} $${}', {
dir: '/',
}),
).resolves.toEqual({
applied: true,
value: 'foo ${ESCAPE_ME} $${ESCAPE_ME_TOO} ${}',
packages/config-loader/src/lib/transform/substitution.ts → packages/config-loader/src/sources/transform/substitution.ts
+5 -2
View File
@@ -15,14 +15,17 @@
*/
import { JsonValue } from '@backstage/types';
import { TransformFunc, EnvFunc } from './types';
import { TransformFunc } from './types';
import { SubstitutionFunc } from '../types';
/**
* A environment variable substitution transform that transforms e.g. 'token ${MY_TOKEN}'
* to 'token abc' if MY_TOKEN is 'abc'. If any of the substituted variables are undefined,
* the entire expression ends up undefined.
*/
export function createSubstitutionTransform(env: EnvFunc): TransformFunc {
export function createSubstitutionTransform(
env: SubstitutionFunc,
): TransformFunc {
return async (input: JsonValue) => {
if (typeof input !== 'string') {
return { applied: false };
packages/config-loader/src/lib/transform/types.ts → packages/config-loader/src/sources/transform/types.ts
+6 -4
View File
@@ -16,13 +16,15 @@
import { JsonValue } from '@backstage/types';
export type EnvFunc = (name: string) => Promise<string | undefined>;
export type ReadFileFunc = (path: string) => Promise<string>;
export interface TransformContext {
dir?: string;
}
export type TransformFunc = (
value: JsonValue,
baseDir: string,
context: TransformContext,
) => Promise<
| {
applied: false;
@@ -30,6 +32,6 @@ export type TransformFunc = (
| {
applied: true;
value: JsonValue | undefined;
newBaseDir?: string | undefined;
newDir?: string | undefined;
}
>;
packages/config-loader/src/lib/transform/utils.ts → packages/config-loader/src/sources/transform/utils.ts
View File
packages/config-loader/src/sources/types.ts
+94
View File
@@ -0,0 +1,94 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AppConfig } from '@backstage/config';
/**
* The data returned by {@link ConfigSource.readConfigData}.
*
* @public
*/
export interface ConfigSourceData extends AppConfig {
/**
* The file path that this configuration was loaded from, if it was loaded from a file.
*/
path?: string;
}
/**
* Options for {@link ConfigSource.readConfigData}.
*
* @public
*/
export interface ReadConfigDataOptions {
signal?: AbortSignal;
}
/**
* The type of the iterator returned by {@link ConfigSource.readConfigData}.
*
* @public
*/
export interface AsyncConfigSourceIterator
extends AsyncIterator<{ configs: ConfigSourceData[] }, void, void> {
[Symbol.asyncIterator](): AsyncIterator<
{ configs: ConfigSourceData[] },
void,
void
>;
}
/**
* A source of configuration data.
*
* @remarks
*
* It is recommended to implement the `readConfigData` method as an async generator.
*
* @example
*
* ```ts
* class MyConfigSource implements ConfigSource {
* async *readConfigData() {
* yield {
* config: [{
* context: 'example',
* data: { backend: { baseUrl: 'http://localhost' } }
* }]
* };
* }
* }
* ```
*
* @public
*/
export interface ConfigSource {
readConfigData(options?: ReadConfigDataOptions): AsyncConfigSourceIterator;
}
/**
* A custom function to be used for substitution withing configuration files.
*
* @remarks
*
* Substitutions use the following syntax: `baseUrl: https://${HOSTNAME}`, where
* `'HOSTNAME'` is the name of the variable to be substituted.
*
* The default substitution function will read the value of the environment.
*
* @public
*/
export type SubstitutionFunc = (name: string) => Promise<string | undefined>;
packages/config-loader/src/sources/utils.ts
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** @internal */
export interface SimpleDeferred<T> {
promise: Promise<T>;
resolve(value: T): void;
}
/** @internal */
export function simpleDefer<T>(): SimpleDeferred<T> {
let resolve: (value: T) => void;
const promise = new Promise<T>(_resolve => {
resolve = _resolve;
});
return { promise, resolve: resolve! };
}
/** @internal */
export async function waitOrAbort<T>(
promise: PromiseLike<T>,
signal?: AbortSignal | Array<AbortSignal | undefined>,
): Promise<[ok: true, value: T] | [ok: false]> {
const signals = [signal].flat().filter((x): x is AbortSignal => !!x);
return new Promise((resolve, reject) => {
if (signals.some(s => s.aborted)) {
resolve([false]);
}
const onAbort = () => {
resolve([false]);
};
promise.then(
value => {
resolve([true, value]);
signals.forEach(s => s.removeEventListener('abort', onAbort));
},
error => {
reject(error);
signals.forEach(s => s.removeEventListener('abort', onAbort));
},
);
signals.forEach(s => s.addEventListener('abort', onAbort));
});
}
yarn.lock
+4
View File
@@ -3923,6 +3923,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "@backstage/config-loader@workspace:packages/config-loader"
dependencies:
"@backstage/backend-test-utils": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/cli-common": "workspace:^"
"@backstage/config": "workspace:^"
@@ -3939,12 +3940,15 @@ __metadata:
json-schema: ^0.4.0
json-schema-merge-allof: ^0.8.1
json-schema-traverse: ^1.0.0
lodash: ^4.17.21
minimist: ^1.2.5
mock-fs: ^5.1.0
msw: ^1.0.0
node-fetch: ^2.6.7
typescript-json-schema: ^0.55.0
yaml: ^2.0.0
yup: ^0.32.9
zen-observable: ^0.10.0
languageName: unknown
linkType: soft