backend-app-api: forklift config implementation from backend-common

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
2023-01-10 16:43:28 +01:00
parent 61ee43ad5b
commit fdc1d3d8ef
4 changed files with 12 additions and 0 deletions
@@ -36,10 +36,14 @@
    "@backstage/backend-common": "workspace:^",
    "@backstage/backend-plugin-api": "workspace:^",
    "@backstage/backend-tasks": "workspace:^",
+    "@backstage/cli-common": "workspace:^",
    "@backstage/config": "workspace:^",
+    "@backstage/config-loader": "workspace:^",
    "@backstage/errors": "workspace:^",
    "@backstage/plugin-auth-node": "workspace:^",
    "@backstage/plugin-permission-node": "workspace:^",
+    "@backstage/types": "workspace:^",
+    "@manypkg/get-packages": "^1.1.3",
    "@types/cors": "^2.8.6",
    "@types/express": "^4.17.6",
    "compression": "^1.7.4",
@@ -50,6 +54,7 @@
    "helmet": "^6.0.0",
    "lodash": "^4.17.21",
    "minimatch": "^5.0.0",
+    "minimist": "^1.2.5",
    "morgan": "^1.10.0",
    "node-forge": "^1.3.1",
    "selfsigned": "^2.0.0",
@@ -61,6 +66,7 @@
    "@types/compression": "^1.7.0",
    "@types/fs-extra": "^9.0.3",
    "@types/http-errors": "^2.0.0",
+    "@types/minimist": "^1.2.0",
    "@types/morgan": "^1.9.0",
    "@types/node-forge": "^1.3.0",
    "@types/stoppable": "^1.1.0",
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LoggerService } from '@backstage/backend-plugin-api';
+import { ConfigReader } from '@backstage/config';
+import { ObservableConfigProxy } from './config';
+
+describe('ObservableConfigProxy', () => {
+  const errLogger = {
+    error: (message: string) => {
+      throw new Error(message);
+    },
+  } as unknown as LoggerService;
+
+  it('should notify subscribers', () => {
+    const config = new ObservableConfigProxy(errLogger);
+
+    const fn = jest.fn();
+    const sub = config.subscribe(fn);
+    expect(config.getOptionalNumber('x')).toBe(undefined);
+
+    config.setConfig(new ConfigReader({}));
+    expect(fn).toHaveBeenCalledTimes(1);
+    expect(config.getOptionalNumber('x')).toBe(undefined);
+
+    config.setConfig(new ConfigReader({ x: 1 }));
+    expect(fn).toHaveBeenCalledTimes(2);
+    expect(config.getOptionalNumber('x')).toBe(1);
+
+    config.setConfig(new ConfigReader({ x: 3 }));
+    expect(fn).toHaveBeenCalledTimes(3);
+    sub.unsubscribe();
+    expect(config.getOptionalNumber('x')).toBe(3);
+
+    config.setConfig(new ConfigReader({ x: 5 }));
+    expect(fn).toHaveBeenCalledTimes(3);
+    expect(config.getOptionalNumber('x')).toBe(5);
+  });
+
+  it('should forward subscriptions', () => {
+    const config1 = new ObservableConfigProxy(errLogger);
+
+    const fn1 = jest.fn();
+    const fn2 = jest.fn();
+    const fn3 = jest.fn();
+    const config2 = config1.getConfig('a');
+    const config3 = config2.getConfig('b');
+    const sub1 = config1.subscribe(fn1);
+    const sub2 = config2.subscribe!(fn2);
+    const sub3 = config3.subscribe!(fn3);
+    expect(config1.getOptionalNumber('x')).toBe(undefined);
+    expect(config2.getOptionalNumber('x')).toBe(undefined);
+    expect(config3.getOptionalNumber('x')).toBe(undefined);
+
+    config1.setConfig(new ConfigReader({}));
+    expect(fn1).toHaveBeenCalledTimes(1);
+    expect(fn2).toHaveBeenCalledTimes(1);
+    expect(fn3).toHaveBeenCalledTimes(1);
+    expect(config1.getOptionalNumber('x')).toBe(undefined);
+    expect(config2.getOptionalNumber('x')).toBe(undefined);
+    expect(config3.getOptionalNumber('x')).toBe(undefined);
+
+    config1.setConfig(new ConfigReader({ x: 1, a: { x: 2, b: { x: 3 } } }));
+    expect(fn1).toHaveBeenCalledTimes(2);
+    expect(fn2).toHaveBeenCalledTimes(2);
+    expect(fn3).toHaveBeenCalledTimes(2);
+    expect(config1.getNumber('x')).toBe(1);
+    expect(config2.getNumber('x')).toBe(2);
+    expect(config3.getNumber('x')).toBe(3);
+
+    sub1.unsubscribe();
+    sub2.unsubscribe();
+    sub3.unsubscribe();
+
+    config1.setConfig(new ConfigReader({ x: 4, a: { x: 5, b: { x: 6 } } }));
+    expect(fn1).toHaveBeenCalledTimes(2);
+    expect(fn2).toHaveBeenCalledTimes(2);
+    expect(fn3).toHaveBeenCalledTimes(2);
+    expect(config1.getNumber('x')).toBe(4);
+    expect(config2.getNumber('x')).toBe(5);
+    expect(config3.getNumber('x')).toBe(6);
+
+    config1.setConfig(new ConfigReader({}));
+    expect(() => config1.getNumber('x')).toThrow(
+      "Missing required config value at 'x'",
+    );
+    expect(() => config2.getNumber('x')).toThrow(
+      "Missing required config value at 'a'",
+    );
+    expect(() => config3.getNumber('x')).toThrow(
+      "Missing required config value at 'a'",
+    );
+
+    config1.setConfig(
+      new ConfigReader({ x: 's', a: { x: 's', b: { x: 's' } } }),
+    );
+    expect(() => config1.getNumber('x')).toThrow(
+      "Unable to convert config value for key 'x' in 'mock-config' to a number",
+    );
+    expect(() => config2.getNumber('x')).toThrow(
+      "Unable to convert config value for key 'a.x' in 'mock-config' to a number",
+    );
+    expect(() => config3.getNumber('x')).toThrow(
+      "Unable to convert config value for key 'a.b.x' in 'mock-config' to a number",
+    );
+  });
+
+  it('should make sub configs available as expected', () => {
+    const config = new ObservableConfigProxy(errLogger);
+
+    config.setConfig(new ConfigReader({ a: { x: 1 } }));
+
+    expect(config.getConfig('a')).toBeDefined();
+    expect(config.getConfig('a').getNumber('x')).toBe(1);
+    expect(config.getConfig('a').getOptionalNumber('x')).toBe(1);
+    expect(config.getOptionalConfig('a')?.getNumber('x')).toBe(1);
+    expect(config.getOptionalConfig('a')?.getOptionalNumber('x')).toBe(1);
+    expect(config.getOptionalConfig('b')).toBeUndefined();
+    expect(() => config.getConfig('b')).toBeDefined();
+    expect(() => config.getConfig('b').get()).toThrow();
+  });
+});
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { resolve as resolvePath } from 'path';
+import parseArgs from 'minimist';
+import { LoggerService } from '@backstage/backend-plugin-api';
+import { findPaths } from '@backstage/cli-common';
+import {
+  loadConfigSchema,
+  loadConfig,
+  ConfigSchema,
+  ConfigTarget,
+  LoadConfigOptionsRemote,
+} from '@backstage/config-loader';
+import { AppConfig, Config, ConfigReader } from '@backstage/config';
+import { JsonValue } from '@backstage/types';
+import { getPackages } from '@manypkg/get-packages';
+
+import { isValidUrl } from './urls';
+
+import { setRootLoggerRedactionList } from './logging/rootLogger';
+
+// Fetch the schema and get all the secrets to pass to the rootLogger for redaction
+const updateRedactionList = (
+  schema: ConfigSchema,
+  configs: AppConfig[],
+  logger: LoggerService,
+) => {
+  const secretAppConfigs = schema.process(configs, {
+    visibility: ['secret'],
+    ignoreSchemaErrors: true,
+  });
+  const secretConfig = ConfigReader.fromConfigs(secretAppConfigs);
+  const values = new Set<string>();
+  const data = secretConfig.get();
+
+  JSON.parse(
+    JSON.stringify(data),
+    (_, v) => typeof v === 'string' && values.add(v),
+  );
+
+  logger.info(
+    `${values.size} secret${
+      values.size > 1 ? 's' : ''
+    } found in the config which will be redacted`,
+  );
+
+  setRootLoggerRedactionList(Array.from(values));
+};
+
+export class ObservableConfigProxy implements Config {
+  private config: Config = new ConfigReader({});
+
+  private readonly subscribers: (() => void)[] = [];
+
+  constructor(
+    private readonly logger: LoggerService,
+    private readonly parent?: ObservableConfigProxy,
+    private parentKey?: string,
+  ) {
+    if (parent && !parentKey) {
+      throw new Error('parentKey is required if parent is set');
+    }
+  }
+
+  setConfig(config: Config) {
+    if (this.parent) {
+      throw new Error('immutable');
+    }
+    this.config = config;
+    for (const subscriber of this.subscribers) {
+      try {
+        subscriber();
+      } catch (error) {
+        this.logger.error(`Config subscriber threw error, ${error}`);
+      }
+    }
+  }
+
+  subscribe(onChange: () => void): { unsubscribe: () => void } {
+    if (this.parent) {
+      return this.parent.subscribe(onChange);
+    }
+
+    this.subscribers.push(onChange);
+    return {
+      unsubscribe: () => {
+        const index = this.subscribers.indexOf(onChange);
+        if (index >= 0) {
+          this.subscribers.splice(index, 1);
+        }
+      },
+    };
+  }
+
+  private select(required: true): Config;
+  private select(required: false): Config | undefined;
+  private select(required: boolean): Config | undefined {
+    if (this.parent && this.parentKey) {
+      if (required) {
+        return this.parent.select(true).getConfig(this.parentKey);
+      }
+      return this.parent.select(false)?.getOptionalConfig(this.parentKey);
+    }
+
+    return this.config;
+  }
+
+  has(key: string): boolean {
+    return this.select(false)?.has(key) ?? false;
+  }
+  keys(): string[] {
+    return this.select(false)?.keys() ?? [];
+  }
+  get<T = JsonValue>(key?: string): T {
+    return this.select(true).get(key);
+  }
+  getOptional<T = JsonValue>(key?: string): T | undefined {
+    return this.select(false)?.getOptional(key);
+  }
+  getConfig(key: string): Config {
+    return new ObservableConfigProxy(this.logger, this, key);
+  }
+  getOptionalConfig(key: string): Config | undefined {
+    if (this.select(false)?.has(key)) {
+      return new ObservableConfigProxy(this.logger, this, key);
+    }
+    return undefined;
+  }
+  getConfigArray(key: string): Config[] {
+    return this.select(true).getConfigArray(key);
+  }
+  getOptionalConfigArray(key: string): Config[] | undefined {
+    return this.select(false)?.getOptionalConfigArray(key);
+  }
+  getNumber(key: string): number {
+    return this.select(true).getNumber(key);
+  }
+  getOptionalNumber(key: string): number | undefined {
+    return this.select(false)?.getOptionalNumber(key);
+  }
+  getBoolean(key: string): boolean {
+    return this.select(true).getBoolean(key);
+  }
+  getOptionalBoolean(key: string): boolean | undefined {
+    return this.select(false)?.getOptionalBoolean(key);
+  }
+  getString(key: string): string {
+    return this.select(true).getString(key);
+  }
+  getOptionalString(key: string): string | undefined {
+    return this.select(false)?.getOptionalString(key);
+  }
+  getStringArray(key: string): string[] {
+    return this.select(true).getStringArray(key);
+  }
+  getOptionalStringArray(key: string): string[] | undefined {
+    return this.select(false)?.getOptionalStringArray(key);
+  }
+}
+
+// A global used to ensure that only a single file watcher is active at a time.
+let currentCancelFunc: () => void;
+
+/**
+ * Load configuration for a Backend.
+ *
+ * This function should only be called once, during the initialization of the backend.
+ *
+ * @public
+ */
+export async function loadBackendConfig(options: {
+  logger: LoggerService;
+  // process.argv or any other overrides
+  remote?: LoadConfigOptionsRemote;
+  argv: string[];
+}): Promise<Config> {
+  const args = parseArgs(options.argv);
+
+  const configTargets: ConfigTarget[] = [args.config ?? []]
+    .flat()
+    .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) }));
+
+  /* eslint-disable-next-line no-restricted-syntax */
+  const paths = findPaths(__dirname);
+
+  // TODO(hhogg): This is fetching _all_ of the packages of the monorepo
+  // in order to find the secrets for redactions, however we only care about
+  // the backend ones, we need to find a way to exclude the frontend packages.
+  const { packages } = await getPackages(paths.targetDir);
+  const schema = await loadConfigSchema({
+    dependencies: packages.map(p => p.packageJson.name),
+  });
+
+  const config = new ObservableConfigProxy(options.logger);
+  const { appConfigs } = await loadConfig({
+    configRoot: paths.targetRoot,
+    configTargets: configTargets,
+    remote: options.remote,
+    watch: {
+      onChange(newConfigs) {
+        options.logger.info(
+          `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`,
+        );
+
+        config.setConfig(ConfigReader.fromConfigs(newConfigs));
+      },
+      stopSignal: new Promise(resolve => {
+        if (currentCancelFunc) {
+          currentCancelFunc();
+        }
+        currentCancelFunc = resolve;
+
+        // For reloads of this module we need to use a dispose handler rather than the global.
+        if (module.hot) {
+          module.hot.addDisposeHandler(resolve);
+        }
+      }),
+    },
+  });
+
+  options.logger.info(
+    `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`,
+  );
+
+  config.setConfig(ConfigReader.fromConfigs(appConfigs));
+
+  // Subscribe to config changes and update the redaction list for logging
+  updateRedactionList(schema, appConfigs, options.logger);
+  config.subscribe(() =>
+    updateRedactionList(schema, appConfigs, options.logger),
+  );
+
+  return config;
+}