From b42531cfedc94bbdc4fa8a445f4a2b2a341bd395 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 13 Apr 2021 13:53:43 +0200 Subject: [PATCH] Support configuration of file storage for SQLite databases Signed-off-by: Oliver Sand --- .changeset/six-turtles-sip.md | 6 +++ packages/backend-common/config.d.ts | 2 +- .../backend-common/src/database/connection.ts | 2 +- .../src/database/sqlite3.test.ts | 36 +++++++++++++-- .../backend-common/src/database/sqlite3.ts | 45 +++++++++++++++++-- 5 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 .changeset/six-turtles-sip.md diff --git a/.changeset/six-turtles-sip.md b/.changeset/six-turtles-sip.md new file mode 100644 index 0000000000..e71b04e64d --- /dev/null +++ b/.changeset/six-turtles-sip.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +--- + +Support configuration of file storage for SQLite databases. Every plugin has its +own database file at the specified path. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 74c199e737..41845f952e 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -57,7 +57,7 @@ export interface Config { database: | { client: 'sqlite3'; - connection: ':memory:' | string; + connection: ':memory:' | string | { filename: string }; } | { client: 'pg'; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 17ef2c461d..3502e21674 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -37,7 +37,7 @@ export function createDatabaseClient( if (client === 'pg') { return createPgDatabaseClient(dbConfig, overrides); } else if (client === 'sqlite3') { - return createSqliteDatabaseClient(dbConfig); + return createSqliteDatabaseClient(dbConfig, overrides); } return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)); diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/sqlite3.test.ts index a6b8e5d84d..3066cb6c9d 100644 --- a/packages/backend-common/src/database/sqlite3.test.ts +++ b/packages/backend-common/src/database/sqlite3.test.ts @@ -25,15 +25,23 @@ describe('sqlite3', () => { new ConfigReader({ client: 'sqlite3', connection }); describe('buildSqliteDatabaseConfig', () => { - it('buidls a string connection', () => { + it('builds an in memory connection', () => { expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({ client: 'sqlite3', - connection: ':memory:', + connection: { filename: ':memory:' }, useNullAsDefault: true, }); }); - it('builds a filename connection', () => { + it('builds a persistent connection, normalize config with filename', () => { + expect(buildSqliteDatabaseConfig(createConfig('/path/to/foo'))).toEqual({ + client: 'sqlite3', + connection: { filename: '/path/to/foo' }, + useNullAsDefault: true, + }); + }); + + it('builds a persistent connection', () => { expect( buildSqliteDatabaseConfig( createConfig({ @@ -49,6 +57,28 @@ describe('sqlite3', () => { }); }); + it('builds a persistent connection per database', () => { + expect( + buildSqliteDatabaseConfig( + createConfig({ + filename: '/path/to/foo', + }), + { + connection: { + database: 'my-database', + }, + }, + ), + ).toEqual({ + client: 'sqlite3', + connection: { + filename: '/path/to/foo/my-database.sqlite', + database: 'my-database', + }, + useNullAsDefault: true, + }); + }); + it('replaces the connection with an override', () => { expect( buildSqliteDatabaseConfig(createConfig(':memory:'), { diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index f5742c68a0..9250315413 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -14,8 +14,10 @@ * limitations under the License. */ -import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; +import fs from 'fs'; +import knexFactory, { Knex } from 'knex'; +import path from 'path'; import { mergeDatabaseConfig } from './config'; /** @@ -29,6 +31,20 @@ export function createSqliteDatabaseClient( overrides?: Knex.Config, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); + + // If storage on disk is used, ensure that the directory exists + if ( + typeof knexConfig.connection === 'object' && + (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename + ) { + const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; + const directory = path.dirname(filename); + + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } + } + const database = knexFactory(knexConfig); database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { @@ -47,12 +63,33 @@ export function createSqliteDatabaseClient( export function buildSqliteDatabaseConfig( dbConfig: Config, overrides?: Knex.Config, -) { - return mergeDatabaseConfig( - dbConfig.get(), +): Knex.Config { + const baseConfig = dbConfig.get(); + + // Normalize config to always contain a connection object + if (typeof baseConfig.connection === 'string') { + baseConfig.connection = { filename: baseConfig.connection }; + } + + const config: Knex.Config = mergeDatabaseConfig( + baseConfig, { useNullAsDefault: true, }, overrides, ); + + // If we don't create an in-memory database, interpret the connection string + // as a directory that contains multiple sqlite files based on the database + // name. + if (config.connection && typeof config.connection === 'object') { + const database = (config.connection as Knex.ConnectionConfig).database; + const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; + + if (database && sqliteConnection.filename !== ':memory:') { + sqliteConnection.filename = `${sqliteConnection.filename}/${database}.sqlite`; + } + } + + return config; }