Merge pull request #2697 from lowjoel/database-refactor

refactor: Expose types to handle Database management

CHANGELOG.md

@@ -8,6 +8,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re
 
 > Collect changes for the next release below
 
+### Backend (example-backend, or backends created with @backstage/create-app)
+
+- A plugin database manager has been created, and plugins can now accept that interface as an argument during initialisation. Notably, the `auth` plugin has a [`createRouter` signature change](./plugins/auth-backend/src/service/router.ts). See [packages/backend/src/index.ts](./packages/backend/src/index.ts) on how to set it up. [#2697](https://github.com/spotify/backstage/pull/2697)
+
 ## v0.1.1-alpha.24
 
 ### Backend (example-backend, or backends created with @backstage/create-app)

packages/backend-common/src/database/SingleConnection.test.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { createDatabaseClient } from './connection';
+import { SingleConnectionDatabaseManager } from './SingleConnection';
+
+jest.mock('./connection');
+
+describe('SingleConnectionDatabaseManager', () => {
+  const createConfig = (data: any) =>
+    ConfigReader.fromConfigs([
+      {
+        context: '',
+        data,
+      },
+    ]);
+
+  const defaultConfigOptions = {
+    backend: {
+      database: {
+        client: 'pg',
+        connection: {
+          host: 'localhost',
+          user: 'foo',
+          password: 'bar',
+          database: 'foodb',
+        },
+      },
+    },
+  };
+  const defaultConfig = () => createConfig(defaultConfigOptions);
+
+  // This is similar to the ts-jest `mocked` helper.
+  const mocked = (f: Function) => f as jest.Mock;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('SingleConnectionDatabaseManager.fromConfig', () => {
+    it('accesses the backend.database key', () => {
+      const getConfig = jest.fn();
+      const config = defaultConfig();
+      config.getConfig = getConfig;
+
+      SingleConnectionDatabaseManager.fromConfig(config);
+
+      expect(getConfig.mock.calls[0][0]).toEqual('backend.database');
+    });
+  });
+
+  describe('SingleConnectionDatabaseManager.forPlugin', () => {
+    const manager = SingleConnectionDatabaseManager.fromConfig(defaultConfig());
+
+    it('connects to a database scoped to the plugin', async () => {
+      const pluginId = 'test1';
+      await manager.forPlugin(pluginId).getClient();
+
+      expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1);
+
+      const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
+      const callArgs = mockCalls[0];
+      expect(callArgs[0].get()).toEqual(defaultConfigOptions.backend.database);
+      expect(callArgs[1].connection.database).toEqual(
+        `backstage_plugin_${pluginId}`,
+      );
+    });
+
+    it('provides different plugins different databases', async () => {
+      const plugin1Id = 'test1';
+      const plugin2Id = 'test2';
+      await manager.forPlugin(plugin1Id).getClient();
+      await manager.forPlugin(plugin2Id).getClient();
+
+      expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(2);
+
+      const mockCalls = mocked(createDatabaseClient).mock.calls;
+      const plugin1CallArgs = mockCalls[0];
+      const plugin2CallArgs = mockCalls[1];
+      expect(plugin1CallArgs[1].connection.database).not.toEqual(
+        plugin2CallArgs[1].connection.database,
+      );
+    });
+  });
+});

packages/backend-common/src/database/SingleConnection.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Knex from 'knex';
+import { Config } from '@backstage/config';
+import { createDatabaseClient, ensureDatabaseExists } from './connection';
+import { PluginDatabaseManager } from './types';
+
+/**
+ * Implements a Database Manager which will automatically create new databases
+ * for plugins when requested. All requested databases are created with the
+ * credentials provided; if the database already exists no attempt to create
+ * the database will be made.
+ */
+export class SingleConnectionDatabaseManager {
+  /**
+   * Creates a new SingleConnectionDatabaseManager instance by reading from the `backend`
+   * config section, specifically the `.database` key for discovering the management
+   * database configuration.
+   *
+   * @param config The loaded application configuration.
+   */
+  static fromConfig(config: Config): SingleConnectionDatabaseManager {
+    return new SingleConnectionDatabaseManager(
+      config.getConfig('backend.database'),
+    );
+  }
+
+  private constructor(private readonly config: Config) {}
+
+  /**
+   * Generates a PluginDatabaseManager for consumption by plugins.
+   *
+   * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique.
+   */
+  forPlugin(pluginId: string): PluginDatabaseManager {
+    const _this = this;
+
+    return {
+      getClient(): Promise<Knex> {
+        return _this.getDatabase(pluginId);
+      },
+    };
+  }
+
+  private async getDatabase(pluginId: string): Promise<Knex> {
+    const config = this.config;
+    const overrides = SingleConnectionDatabaseManager.getDatabaseOverrides(
+      pluginId,
+    );
+    const overrideConfig = overrides.connection as Knex.ConnectionConfig;
+    await this.ensureDatabase(overrideConfig.database);
+
+    return createDatabaseClient(config, overrides);
+  }
+
+  private static getDatabaseOverrides(pluginId: string): Knex.Config {
+    return {
+      connection: {
+        database: `backstage_plugin_${pluginId}`,
+      },
+    };
+  }
+
+  private async ensureDatabase(database: string) {
+    const config = this.config;
+    await ensureDatabaseExists(config, database);
+  }
+}

packages/backend-common/src/database/config.test.ts

@@ -18,6 +18,18 @@ import { mergeDatabaseConfig } from './config';
 
 describe('config', () => {
   describe(mergeDatabaseConfig, () => {
+    it('does not mutate the input object', () => {
+      const input = {
+        original: 'key',
+      };
+      const override = {
+        added: 'value',
+      };
+
+      mergeDatabaseConfig(input, override);
+      expect(input).not.toHaveProperty('added');
+    });
+
     it('does not require overrides', () => {
       expect(
         mergeDatabaseConfig({

packages/backend-common/src/database/config.ts

@@ -19,9 +19,9 @@ import { merge } from 'lodash';
 /**
  * Merges database objects together
  *
- * @param config The base config
+ * @param config The base config. The input is not modified
  * @param overrides Any additional overrides
  */
 export function mergeDatabaseConfig(config: any, ...overrides: any[]) {
-  return merge(config, ...overrides);
+  return merge({}, config, ...overrides);
 }

packages/backend-common/src/database/index.ts

@@ -15,3 +15,5 @@
  */
 
 export * from './connection';
+export * from './types';
+export * from './SingleConnection';

packages/backend-common/src/database/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import knex from 'knex';
+
+/**
+ * The PluginDatabaseManager manages access to databases that Plugins get.
+ */
+export interface PluginDatabaseManager {
+  /**
+   * getClient provides backend plugins database connections for itself.
+   *
+   * The purpose of this method is to allow plugins to get isolated data
+   * stores so that plugins are discouraged from database integration.
+   */
+  getClient(): Promise<knex>;
+}

packages/backend/src/index.ts

@@ -24,17 +24,16 @@
 
 import Router from 'express-promise-router';
 import {
-  ensureDatabaseExists,
-  createDatabaseClient,
   createServiceBuilder,
   loadBackendConfig,
   getRootLogger,
   useHotMemoize,
   notFoundHandler,
+  SingleConnectionDatabaseManager,
   SingleHostDiscovery,
   UrlReaders,
 } from '@backstage/backend-common';
-import { ConfigReader, AppConfig } from '@backstage/config';
+import { ConfigReader } from '@backstage/config';
 import healthcheck from './plugins/healthcheck';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
@@ -48,24 +47,18 @@ import graphql from './plugins/graphql';
 import app from './plugins/app';
 import { PluginEnvironment } from './types';
 
-function makeCreateEnv(loadedConfigs: AppConfig[]) {
-  const config = ConfigReader.fromConfigs(loadedConfigs);
+function makeCreateEnv(config: ConfigReader) {
   const root = getRootLogger();
   const reader = UrlReaders.default({ logger: root, config });
   const discovery = SingleHostDiscovery.fromConfig(config);
 
   root.info(`Created UrlReader ${reader}`);
 
+  const databaseManager = SingleConnectionDatabaseManager.fromConfig(config);
+
   return (plugin: string): PluginEnvironment => {
     const logger = root.child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
-      config.getConfig('backend.database'),
-      {
-        connection: {
-          database: `backstage_plugin_${plugin}`,
-        },
-      },
-    );
+    const database = databaseManager.forPlugin(plugin);
     return { logger, database, config, reader, discovery };
   };
 }
@@ -73,12 +66,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
 async function main() {
   const configs = await loadBackendConfig();
   const configReader = ConfigReader.fromConfigs(configs);
-  const createEnv = makeCreateEnv(configs);
-  await ensureDatabaseExists(
-    configReader.getConfig('backend.database'),
-    'backstage_plugin_catalog',
-    'backstage_plugin_auth',
-  );
+  const createEnv = makeCreateEnv(configReader);
 
   const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck'));
   const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));

packages/backend/src/plugins/catalog.ts

@@ -34,7 +34,9 @@ export default async function createPlugin({
 }: PluginEnvironment) {
   const locationReader = new LocationReaders({ logger, reader, config });
 
-  const db = await DatabaseManager.createDatabase(database, { logger });
+  const db = await DatabaseManager.createDatabase(await database.getClient(), {
+    logger,
+  });
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);
   const locationsCatalog = new DatabaseLocationsCatalog(db);
   const higherOrderOperation = new HigherOrderOperations(

packages/backend/src/types.ts

@@ -14,14 +14,17 @@
  * limitations under the License.
  */
 
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
+import {
+  PluginDatabaseManager,
+  PluginEndpointDiscovery,
+  UrlReader,
+} from '@backstage/backend-common';
 
 export type PluginEnvironment = {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseManager;
   config: Config;
   reader: UrlReader;
   discovery: PluginEndpointDiscovery;

packages/create-app/templates/default-app/packages/backend/src/index.ts

@@ -8,17 +8,16 @@
 
 import Router from 'express-promise-router';
 import {
-  ensureDatabaseExists,
-  createDatabaseClient,
   createServiceBuilder,
   loadBackendConfig,
   getRootLogger,
   useHotMemoize,
   notFoundHandler,
+  SingleConnectionDatabaseManager,
   SingleHostDiscovery,
   UrlReaders,
 } from '@backstage/backend-common';
-import { ConfigReader, AppConfig } from '@backstage/config';
+import { ConfigReader } from '@backstage/config';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
 import scaffolder from './plugins/scaffolder';
@@ -26,24 +25,18 @@ import proxy from './plugins/proxy';
 import techdocs from './plugins/techdocs';
 import { PluginEnvironment } from './types';
 
-function makeCreateEnv(loadedConfigs: AppConfig[]) {
-  const config = ConfigReader.fromConfigs(loadedConfigs);
+function makeCreateEnv(config: ConfigReader) {
   const root = getRootLogger();
   const reader = UrlReaders.default({ logger: root, config });
   const discovery = SingleHostDiscovery.fromConfig(config);
 
   root.info(`Created UrlReader ${reader}`);
 
+  const databaseManager = SingleConnectionDatabaseManager.fromConfig(config);
+
   return (plugin: string): PluginEnvironment => {
     const logger = root.child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
-      config.getConfig('backend.database'),
-      {
-        connection: {
-          database: `backstage_plugin_${plugin}`,
-        },
-      },
-    );
+    const database = databaseManager.forPlugin(plugin);
     return { logger, database, config, reader, discovery };
   };
 }
@@ -51,12 +44,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
 async function main() {
   const configs = await loadBackendConfig();
   const configReader = ConfigReader.fromConfigs(configs);
-  const createEnv = makeCreateEnv(configs);
-  await ensureDatabaseExists(
-    configReader.getConfig('backend.database'),
-    'backstage_plugin_catalog',
-    'backstage_plugin_auth',
-  );
+  const createEnv = makeCreateEnv(configReader);
 
   const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
   const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));

packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts

@@ -18,7 +18,9 @@ export default async function createPlugin({
 }: PluginEnvironment) {
   const locationReader = new LocationReaders({ logger, reader, config });
 
-  const db = await DatabaseManager.createDatabase(database, { logger });
+  const db = await DatabaseManager.createDatabase(await database.getClient(),
+    { logger },
+  );
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);
   const locationsCatalog = new DatabaseLocationsCatalog(db);
   const higherOrderOperation = new HigherOrderOperations(

packages/create-app/templates/default-app/packages/backend/src/types.ts

@@ -1,11 +1,14 @@
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
+import {
+  PluginDatabaseManager,
+  PluginEndpointDiscovery,
+  UrlReader,
+} from '@backstage/backend-common';
 
 export type PluginEnvironment = {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseManager;
   config: Config;
   reader: UrlReader
   discovery: PluginEndpointDiscovery;

packages/e2e-test/src/e2e-test.ts

@@ -328,8 +328,8 @@ async function testAppServe(pluginName: string, appDir: string) {
   }
 }
 
-/** Creates PG databases (drops if exists before) */
-async function createDB(database: string) {
+/** Drops PG databases */
+async function dropDB(database: string) {
   const config = {
     host: process.env.POSTGRES_HOST,
     port: process.env.POSTGRES_PORT,
@@ -342,7 +342,6 @@ async function createDB(database: string) {
   } catch (_) {
     /* do nothing*/
   }
-  return pgtools.createdb(config, database);
 }
 
 /**
@@ -350,7 +349,7 @@ async function createDB(database: string) {
  */
 async function testBackendStart(appDir: string, isPostgres: boolean) {
   if (isPostgres) {
-    print('Creating DBs');
+    print('Dropping old DBs');
     await Promise.all(
       [
         'catalog',
@@ -359,7 +358,7 @@ async function testBackendStart(appDir: string, isPostgres: boolean) {
         'identity',
         'proxy',
         'techdocs',
-      ].map(name => createDB(`backstage_plugin_${name}`)),
+      ].map(name => dropDB(`backstage_plugin_${name}`)),
     );
     print('Created DBs');
   }

plugins/auth-backend/src/service/router.ts

@@ -17,7 +17,6 @@
 import express from 'express';
 import Router from 'express-promise-router';
 import cookieParser from 'cookie-parser';
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { createAuthProvider } from '../providers';
 import { Config } from '@backstage/config';
@@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity';
 import {
   NotFoundError,
   PluginEndpointDiscovery,
+  PluginDatabaseManager,
 } from '@backstage/backend-common';
 
 export interface RouterOptions {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseManager;
   config: Config;
   discovery: PluginEndpointDiscovery;
 }
@@ -47,7 +47,9 @@ export async function createRouter({
 
   const keyDurationSeconds = 3600;
 
-  const keyStore = await DatabaseKeyStore.create({ database });
+  const keyStore = await DatabaseKeyStore.create({
+    database: await database.getClient(),
+  });
   const tokenIssuer = new TokenFactory({
     issuer: authUrl,
     keyStore,

plugins/auth-backend/src/service/standaloneServer.ts

@@ -53,7 +53,11 @@ export async function startStandaloneServer(
   const router = await createRouter({
     logger,
     config,
-    database,
+    database: {
+      async getClient() {
+        return database;
+      },
+    },
     discovery,
   });