From efc73db10cd75b6d1746e85fcba0ac39598074b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 11 Mar 2022 14:01:35 +0100 Subject: [PATCH] switch us over to better-sqlite3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/heavy-kangaroos-confess.md | 13 +++ .changeset/warm-impalas-march.md | 24 +++++ app-config.yaml | 2 +- docs/getting-started/configuration.md | 2 +- .../tutorials/configuring-plugin-databases.md | 16 ++-- docs/tutorials/switching-sqlite-postgres.md | 2 +- packages/backend-common/config.d.ts | 4 +- .../src/database/DatabaseManager.test.ts | 28 +++--- .../src/database/config.test.ts | 24 ++++- .../src/database/connection.test.ts | 10 ++- .../src/database/connectors/sqlite3.test.ts | 90 +------------------ packages/backend-common/src/database/util.ts | 1 + .../src/tasks/PluginTaskSchedulerJanitor.ts | 2 +- packages/backend-test-utils/.snyk | 47 ---------- packages/backend-test-utils/package.json | 2 +- .../backend-test-utils/src/database/types.ts | 2 +- packages/backend/.snyk | 47 ---------- packages/backend/knexfile.ts | 2 +- packages/backend/package.json | 2 +- .../templates/default-app/app-config.yaml.hbs | 2 +- .../packages/backend/package.json.hbs | 2 +- .../src/identity/DatabaseKeyStore.test.ts | 2 +- .../src/identity/KeyStores.test.ts | 2 +- .../src/service/standaloneServer.ts | 2 +- .../src/service/standaloneServer.ts | 2 +- plugins/catalog-backend/.snyk | 47 ---------- plugins/catalog-backend/knexfile.js | 2 +- plugins/catalog-backend/package.json | 2 +- .../src/database/DefaultProcessingDatabase.ts | 4 +- .../src/database/conversion.ts | 6 +- .../src/service/standaloneServer.ts | 4 +- .../src/service/CodeCoverageDatabase.test.ts | 2 +- .../src/service/router.test.ts | 2 +- .../src/service/standaloneServer.ts | 2 +- plugins/config-schema/dev/example-schema.json | 2 +- .../tasks/StorageTaskBroker.test.ts | 2 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 2 +- .../src/service/router.test.ts | 2 +- yarn.lock | 12 --- 39 files changed, 121 insertions(+), 302 deletions(-) create mode 100644 .changeset/heavy-kangaroos-confess.md create mode 100644 .changeset/warm-impalas-march.md delete mode 100644 packages/backend-test-utils/.snyk delete mode 100644 packages/backend/.snyk delete mode 100644 plugins/catalog-backend/.snyk diff --git a/.changeset/heavy-kangaroos-confess.md b/.changeset/heavy-kangaroos-confess.md new file mode 100644 index 0000000000..6d39e09db1 --- /dev/null +++ b/.changeset/heavy-kangaroos-confess.md @@ -0,0 +1,13 @@ +--- +'@backstage/backend-common': patch +'@backstage/backend-tasks': patch +'@backstage/backend-test-utils': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-bazaar-backend': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-config-schema': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Use `better-sqlite3` instead of `@vscode/sqlite3` diff --git a/.changeset/warm-impalas-march.md b/.changeset/warm-impalas-march.md new file mode 100644 index 0000000000..6db9900173 --- /dev/null +++ b/.changeset/warm-impalas-march.md @@ -0,0 +1,24 @@ +--- +'@backstage/create-app': patch +--- + +The main repo has switched from `@vscode/sqlite3` to `better-sqlite3` as its preferred SQLite installation. This decision was triggered by a number of issues with the former that arose because it needs build infrastructure in place and functional in order to be installed. The main drawback of this is that the new package uses the database client ID `better-sqlite3` instead of the plain `sqlite3`. + +If you want to perform the same switch in your own repository, + +- Replace all of your `package.json` dependencies on `@vscode/sqlite3` with the latest version of `better-sqlite3` instead + + ```diff + "dependencies": { + - "@vscode/sqlite3": "^5.0.7", + + "better-sqlite3": "^7.5.0", + ``` + +- In your app-config and tests, wherever you supply `client: 'sqlite3'`, instead supply `client: 'better-sqlite3` + + ```diff + backend: + database: + - client: sqlite3 + + client: better-sqlite3 + ``` diff --git a/app-config.yaml b/app-config.yaml index ab68d4398f..b3ca3c7203 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -33,7 +33,7 @@ backend: listen: port: 7007 database: - client: sqlite3 + client: better-sqlite3 connection: ':memory:' cache: store: memory diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index b48bdbd858..3e418fadf0 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -78,7 +78,7 @@ from the previous steps. ```diff backend: database: -- client: sqlite3 +- client: better-sqlite3 - connection: ':memory:' + # config options: https://node-postgres.com/api/client + client: pg diff --git a/docs/tutorials/configuring-plugin-databases.md b/docs/tutorials/configuring-plugin-databases.md index 9277bb5028..3e18b1f30e 100644 --- a/docs/tutorials/configuring-plugin-databases.md +++ b/docs/tutorials/configuring-plugin-databases.md @@ -33,17 +33,17 @@ Backstage's databases. ### Dependencies Please ensure the appropriate database drivers are installed in your `backend` -package. If you intend to use both `postgres` and `sqlite3`, you can install +package. If you intend to use both PostgreSQL and SQLite, you can install both of them. ```sh cd packages/backend -# install pg if you need postgres +# install pg if you need PostgreSQL yarn add pg -# install sqlite3 if you intend to set it as the client -yarn add sqlite3 +# install SQLite 3 if you intend to set it as the client +yarn add better-sqlite3 ``` From an operational perspective, you only need to install drivers for clients @@ -66,14 +66,14 @@ configurations below. ### Minimal In-Memory Configuration -In the example below, we are using `sqlite3` in-memory databases for all +In the example below, we are using `better-sqlite3` in-memory databases for all plugins. You may want to use this configuration for testing or other non-durable use cases. ```yaml backend: database: - client: sqlite3 + client: better-sqlite3 connection: ':memory:' ``` @@ -138,7 +138,7 @@ backend: ### PostgreSQL and SQLite 3 The example below uses PostgreSQL (`pg`) as the database client for all plugins -except the `auth` plugin which uses `sqlite3`. As the `auth` plugin's client +except the `auth` plugin which uses `better-sqlite3`. As the `auth` plugin's client type is different from the base client type, the connection configuration for `auth` is used verbatim without extending the base configuration for PostgreSQL. @@ -149,7 +149,7 @@ backend: connection: 'postgresql://foo:bar@some.example-pg-instance.tld:5432' plugin: auth: - client: sqlite3 + client: better-sqlite3 connection: ':memory:' ``` diff --git a/docs/tutorials/switching-sqlite-postgres.md b/docs/tutorials/switching-sqlite-postgres.md index 8a7f90b21f..df5d78f56d 100644 --- a/docs/tutorials/switching-sqlite-postgres.md +++ b/docs/tutorials/switching-sqlite-postgres.md @@ -33,7 +33,7 @@ configuration for the backend: ```diff backend: database: -- client: sqlite3 +- client: better-sqlite3 - connection: ':memory:' + # config options: https://node-postgres.com/api/client + client: pg diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 0c3e3132c0..058805aa62 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -70,7 +70,7 @@ export interface Config { /** Database connection configuration, select base database type using the `client` field */ database: { /** Default database client to use */ - client: 'sqlite3' | 'pg'; + client: 'better-sqlite3' | 'sqlite3' | 'pg'; /** * Base database connection string or Knex object * @secret @@ -106,7 +106,7 @@ export interface Config { plugin?: { [pluginId: string]: { /** Database client override */ - client?: 'sqlite3' | 'pg'; + client?: 'better-sqlite3' | 'sqlite3' | 'pg'; /** * Database connection string or Knex object override * @secret diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts index a48b1584aa..2b2cc17f91 100644 --- a/packages/backend-common/src/database/DatabaseManager.test.ts +++ b/packages/backend-common/src/database/DatabaseManager.test.ts @@ -97,13 +97,13 @@ describe('DatabaseManager', () => { }, }, differentclient: { - client: 'sqlite3', + client: 'better-sqlite3', connection: { filename: 'plugin_with_different_client', }, }, differentclientconnstring: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, stringoverride: { @@ -176,7 +176,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, @@ -198,7 +198,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: 'some-file-path', }, }, @@ -215,7 +215,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: { directory: 'sqlite-files', }, @@ -239,7 +239,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: { directory: 'sqlite-files', }, @@ -270,7 +270,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: { directory: 'sqlite-files', }, @@ -349,7 +349,7 @@ describe('DatabaseManager', () => { // plugin connection should be used as base config, client is different expect(baseConfig.get()).toMatchObject({ - client: 'sqlite3', + client: 'better-sqlite3', connection: config.backend.database.plugin[pluginId].connection, }); }); @@ -361,10 +361,10 @@ describe('DatabaseManager', () => { const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); const [baseConfig, overrides] = mockCalls[0]; - // plugin client should be sqlite3 - expect(baseConfig.get().client).toEqual('sqlite3'); + // plugin client should be better-sqlite3 + expect(baseConfig.get().client).toEqual('better-sqlite3'); - // sqlite3 uses 'filename' instead of 'database' + // SQLite uses 'filename' instead of 'database' expect(overrides).toHaveProperty( 'connection.filename', 'plugin_with_different_client', @@ -378,7 +378,7 @@ describe('DatabaseManager', () => { const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); const [baseConfig, overrides] = mockCalls[0]; - expect(baseConfig.get().client).toEqual('sqlite3'); + expect(baseConfig.get().client).toEqual('better-sqlite3'); expect(overrides).toHaveProperty('connection.filename', ':memory:'); }); @@ -465,7 +465,7 @@ describe('DatabaseManager', () => { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', pluginDivisionMode: 'schema', connection: { host: 'localhost', @@ -484,7 +484,7 @@ describe('DatabaseManager', () => { const [baseConfig, overrides] = mockCalls[0]; expect(baseConfig.get()).toMatchObject({ - client: 'sqlite3', + client: 'better-sqlite3', connection: config.backend.database.connection, }); diff --git a/packages/backend-common/src/database/config.test.ts b/packages/backend-common/src/database/config.test.ts index 01ec599cf4..a5bf4a79d2 100644 --- a/packages/backend-common/src/database/config.test.ts +++ b/packages/backend-common/src/database/config.test.ts @@ -101,7 +101,7 @@ describe('config', () => { expect( mergeDatabaseConfig( { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, }, @@ -112,7 +112,27 @@ describe('config', () => { }, ), ).toEqual({ - client: 'sqlite3', + client: 'better-sqlite3', + connection: { + filename: '/path/to/file', + }, + useNullAsDefault: true, + }); + expect( + mergeDatabaseConfig( + { + client: 'better-sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }, + { + connection: { + filename: '/path/to/file', + }, + }, + ), + ).toEqual({ + client: 'better-sqlite3', connection: { filename: '/path/to/file', }, diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 6f02e7b2ed..cb65113a9c 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -59,7 +59,7 @@ describe('database connection', () => { expect( createDatabaseClient( new ConfigReader({ - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }), ), @@ -133,7 +133,7 @@ describe('database connection', () => { }); it('returns Knex config for sqlite', () => { - expect(createNameOverride('sqlite3', 'testsqlite')).toHaveProperty( + expect(createNameOverride('better-sqlite3', 'testsqlite')).toHaveProperty( 'connection.filename', 'testsqlite', ); @@ -178,7 +178,9 @@ describe('database connection', () => { }); it('throws error for sqlite', () => { - expect(createSchemaOverride('sqlite3', 'testsqlite')).toBeUndefined(); + expect( + createSchemaOverride('better-sqlite3', 'testsqlite'), + ).toBeUndefined(); }); it('returns Knex config for mysql', () => { @@ -218,7 +220,7 @@ describe('database connection', () => { return expect( ensureSchemaExists( new ConfigReader({ - client: 'sqlite3', + client: 'better-sqlite3', schema: 'catalog', connection: ':memory:', }), diff --git a/packages/backend-common/src/database/connectors/sqlite3.test.ts b/packages/backend-common/src/database/connectors/sqlite3.test.ts index d751d895d0..2bd517d373 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.test.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.test.ts @@ -21,87 +21,6 @@ import { createSqliteDatabaseClient, } from './sqlite3'; -describe('sqlite3', () => { - const createConfig = (connection: any) => - new ConfigReader({ client: 'sqlite3', connection }); - - describe('buildSqliteDatabaseConfig', () => { - it('builds an in-memory connection', () => { - expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({ - client: 'sqlite3', - connection: { filename: ':memory:' }, - useNullAsDefault: true, - }); - }); - - it('builds an in-memory connection by override with filename', () => { - expect( - buildSqliteDatabaseConfig( - createConfig(path.join('path', 'to', 'foo')), - { connection: ':memory:' }, - ), - ).toEqual({ - client: 'sqlite3', - connection: { filename: ':memory:' }, - useNullAsDefault: true, - }); - }); - - it('builds a persistent connection, normalize config with filename', () => { - expect( - buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))), - ).toEqual({ - client: 'sqlite3', - connection: { filename: path.join('path', 'to', 'foo') }, - useNullAsDefault: true, - }); - }); - - it('builds a persistent connection', () => { - expect( - buildSqliteDatabaseConfig( - createConfig({ - filename: path.join('path', 'to', 'foo'), - }), - ), - ).toEqual({ - client: 'sqlite3', - connection: { - filename: path.join('path', 'to', 'foo'), - }, - useNullAsDefault: true, - }); - }); - - it('replaces the connection with an override', () => { - expect( - buildSqliteDatabaseConfig(createConfig(':memory:'), { - connection: { filename: path.join('path', 'to', 'foo') }, - }), - ).toEqual({ - client: 'sqlite3', - connection: { - filename: path.join('path', 'to', 'foo'), - }, - useNullAsDefault: true, - }); - }); - }); - - describe('createSqliteDatabaseClient', () => { - it('creates an in memory knex instance', () => { - expect( - createSqliteDatabaseClient( - createConfig({ - client: 'sqlite3', - connection: ':memory:', - }), - ), - ).toBeTruthy(); - }); - }); -}); - describe('better-sqlite3', () => { const createConfig = (connection: any) => new ConfigReader({ client: 'better-sqlite3', connection }); @@ -171,14 +90,7 @@ describe('better-sqlite3', () => { describe('createSqliteDatabaseClient', () => { it('creates an in memory knex instance', () => { - expect( - createSqliteDatabaseClient( - createConfig({ - client: 'better-sqlite3', - connection: ':memory:', - }), - ), - ).toBeTruthy(); + expect(createSqliteDatabaseClient(createConfig(':memory:'))).toBeTruthy(); }); }); }); diff --git a/packages/backend-common/src/database/util.ts b/packages/backend-common/src/database/util.ts index a96eeb3a7b..0a7d3cd7f7 100644 --- a/packages/backend-common/src/database/util.ts +++ b/packages/backend-common/src/database/util.ts @@ -28,6 +28,7 @@ export function isDatabaseConflictError(e: unknown) { return ( typeof message === 'string' && (/SQLITE_CONSTRAINT(?:_UNIQUE)?: UNIQUE/.test(message) || + /UNIQUE constraint failed:/.test(message) || /unique constraint/.test(message)) ); } diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.ts index c77fe4e925..27555036a1 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.ts @@ -56,7 +56,7 @@ export class PluginTaskSchedulerJanitor { // SQLite currently (Oct 1 2021) returns a number for returning() // statements, effectively ignoring them and instead returning the outcome // of the delete() - and knex also emits a warning about that fact, which - // is why we avoid that entirely for the sqlite3 driver. + // is why we avoid that entirely for the sqlite3 family of drivers. // https://github.com/knex/knex/issues/4370 // https://github.com/mapbox/node-sqlite3/issues/1453 diff --git a/packages/backend-test-utils/.snyk b/packages/backend-test-utils/.snyk deleted file mode 100644 index ad62599e66..0000000000 --- a/packages/backend-test-utils/.snyk +++ /dev/null @@ -1,47 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.22.1 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-JS-TAR-1579155: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579152: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579147: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536758: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536531: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536528: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z -patch: {} diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index fc29c9bc26..85965233d4 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -37,7 +37,7 @@ "@backstage/backend-common": "^0.13.0", "@backstage/cli": "^0.15.2", "@backstage/config": "^0.1.15", - "@vscode/sqlite3": "^5.0.7", + "better-sqlite3": "^7.5.0", "knex": "^1.0.2", "msw": "^0.35.0", "mysql2": "^2.2.5", diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts index 5aba48952f..083853ac78 100644 --- a/packages/backend-test-utils/src/database/types.ts +++ b/packages/backend-test-utils/src/database/types.ts @@ -66,6 +66,6 @@ export const allDatabases: Record = }, SQLITE_3: { name: 'SQLite 3.x', - driver: 'sqlite3', + driver: 'better-sqlite3', }, }); diff --git a/packages/backend/.snyk b/packages/backend/.snyk deleted file mode 100644 index ad62599e66..0000000000 --- a/packages/backend/.snyk +++ /dev/null @@ -1,47 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.22.1 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-JS-TAR-1579155: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579152: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579147: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536758: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536531: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536528: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z -patch: {} diff --git a/packages/backend/knexfile.ts b/packages/backend/knexfile.ts index ca05575153..93993c1d5c 100644 --- a/packages/backend/knexfile.ts +++ b/packages/backend/knexfile.ts @@ -16,7 +16,7 @@ module.exports = { development: { - client: 'sqlite3', + client: 'better-sqlite3', connection: { filename: './dev.sqlite3', }, diff --git a/packages/backend/package.json b/packages/backend/package.json index c65dc54b31..a6f91e036f 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -62,7 +62,7 @@ "@backstage/plugin-todo-backend": "^0.1.26", "@gitbeaker/node": "^35.1.0", "@octokit/rest": "^18.5.3", - "@vscode/sqlite3": "^5.0.7", + "better-sqlite3": "^7.5.0", "azure-devops-node-api": "^11.0.1", "dockerode": "^3.3.1", "example-app": "link:../app", diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 3afed0ed82..02139f8084 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -25,7 +25,7 @@ backend: credentials: true {{#if dbTypeSqlite}} database: - client: sqlite3 + client: better-sqlite3 connection: ':memory:' {{/if}} {{#if dbTypePG}} diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index 79d6772d7e..69a09daef2 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -46,7 +46,7 @@ "pg": "^8.3.0", {{/if}} {{#if dbTypeSqlite}} - "@vscode/sqlite3": "^5.0.7", + "better-sqlite3": "^7.5.0", {{/if}} "winston": "^3.2.1" }, diff --git a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts index bb4c129e41..fa7b251780 100644 --- a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts +++ b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts @@ -20,7 +20,7 @@ import { DateTime } from 'luxon'; function createDB() { const knex = Knex({ - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, }); diff --git a/plugins/auth-backend/src/identity/KeyStores.test.ts b/plugins/auth-backend/src/identity/KeyStores.test.ts index f30263e27f..375db4698b 100644 --- a/plugins/auth-backend/src/identity/KeyStores.test.ts +++ b/plugins/auth-backend/src/identity/KeyStores.test.ts @@ -49,7 +49,7 @@ describe('KeyStores', () => { const config = new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts index 4e416e8dc7..16ebbc9345 100644 --- a/plugins/auth-backend/src/service/standaloneServer.ts +++ b/plugins/auth-backend/src/service/standaloneServer.ts @@ -39,7 +39,7 @@ export async function startStandaloneServer( const database = useHotMemoize(module, () => { const knex = Knex({ - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, }); diff --git a/plugins/bazaar-backend/src/service/standaloneServer.ts b/plugins/bazaar-backend/src/service/standaloneServer.ts index 19b0d68039..e8a49fbf5d 100644 --- a/plugins/bazaar-backend/src/service/standaloneServer.ts +++ b/plugins/bazaar-backend/src/service/standaloneServer.ts @@ -38,7 +38,7 @@ export async function startStandaloneServer( const db = useHotMemoize(module, () => { const knex = knexFactory({ - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, }); diff --git a/plugins/catalog-backend/.snyk b/plugins/catalog-backend/.snyk deleted file mode 100644 index ad62599e66..0000000000 --- a/plugins/catalog-backend/.snyk +++ /dev/null @@ -1,47 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.22.1 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-JS-TAR-1579155: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579152: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1579147: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536758: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536531: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z - SNYK-JS-TAR-1536528: - - 'sqlite3 > node-gyp > tar': - reason: >- - The only usage is via node-gyp; there is no unpacking of untrusted tar - files - expires: 2022-11-11T14:30:05.581Z - created: 2021-11-11T14:30:05.582Z -patch: {} diff --git a/plugins/catalog-backend/knexfile.js b/plugins/catalog-backend/knexfile.js index 4c8be42673..4cf9ef77c3 100644 --- a/plugins/catalog-backend/knexfile.js +++ b/plugins/catalog-backend/knexfile.js @@ -17,7 +17,7 @@ // This file makes it possible to run "yarn knex migrate:make some_file_name" // to assist in making new migrations module.exports = { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, migrations: { diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 578bc09d8f..54b19f5269 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -77,7 +77,7 @@ "@types/lodash": "^4.14.151", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", - "@vscode/sqlite3": "^5.0.7", + "better-sqlite3": "^7.5.0", "msw": "^0.35.0", "supertest": "^6.1.3", "wait-for-expect": "^3.0.2", diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 6ffd054435..061be0a767 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -35,7 +35,7 @@ import { ListParentsResult, } from './types'; import { DeferredEntity } from '../processing/types'; -import { RefreshIntervalFunction } from '../processing/refresh'; +import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { @@ -59,7 +59,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { private readonly options: { database: Knex; logger: Logger; - refreshInterval: RefreshIntervalFunction; + refreshInterval: ProcessingIntervalFunction; }, ) { initDatabaseMetrics(options.database); diff --git a/plugins/catalog-backend/src/database/conversion.ts b/plugins/catalog-backend/src/database/conversion.ts index 54b642f3ff..827ae9b367 100644 --- a/plugins/catalog-backend/src/database/conversion.ts +++ b/plugins/catalog-backend/src/database/conversion.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { isDatabaseConflictError } from '@backstage/backend-common'; import { ConflictError, InputError } from '@backstage/errors'; import { DateTime } from 'luxon'; @@ -47,10 +48,7 @@ export function timestampToDateTime(input: Date | string): DateTime { * Rethrows an error, possibly translating it to a more precise error type. */ export function rethrowError(e: any): never { - if ( - /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || - /unique constraint/.test(e.message) - ) { + if (isDatabaseConflictError(e)) { throw new ConflictError(`Rejected due to a conflicting entity`, e); } diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index ba5604a561..e48515fb8b 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -46,7 +46,9 @@ export async function startStandaloneServer( const database = useHotMemoize(module, () => { const manager = DatabaseManager.fromConfig( new ConfigReader({ - backend: { database: { client: 'sqlite3', connection: ':memory:' } }, + backend: { + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, }), ); return manager.forPlugin('catalog'); diff --git a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts index 15028a76fd..635debca02 100644 --- a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts +++ b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts @@ -26,7 +26,7 @@ const db = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/plugins/code-coverage-backend/src/service/router.test.ts b/plugins/code-coverage-backend/src/service/router.test.ts index 4e6a66fdd8..7d1825c25f 100644 --- a/plugins/code-coverage-backend/src/service/router.test.ts +++ b/plugins/code-coverage-backend/src/service/router.test.ts @@ -31,7 +31,7 @@ function createDatabase(): PluginDatabaseManager { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/plugins/code-coverage-backend/src/service/standaloneServer.ts b/plugins/code-coverage-backend/src/service/standaloneServer.ts index 291f78ffc5..c4ced6baeb 100644 --- a/plugins/code-coverage-backend/src/service/standaloneServer.ts +++ b/plugins/code-coverage-backend/src/service/standaloneServer.ts @@ -40,7 +40,7 @@ export async function startStandaloneServer( const db = useHotMemoize(module, () => { const knex = knexFactory({ - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', useNullAsDefault: true, }); diff --git a/plugins/config-schema/dev/example-schema.json b/plugins/config-schema/dev/example-schema.json index be297aaa8d..86f05d500d 100644 --- a/plugins/config-schema/dev/example-schema.json +++ b/plugins/config-schema/dev/example-schema.json @@ -346,7 +346,7 @@ "properties": { "client": { "type": "string", - "enum": ["sqlite3"] + "enum": ["sqlite3", "better-sqlite3"] }, "connection": { "type": "string" diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 52fd175dc8..4cac2206fb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -26,7 +26,7 @@ async function createStore(): Promise { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 1a099e4c53..e1421486f4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -33,7 +33,7 @@ async function createStore(): Promise { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 41bc73c7ae..6caed7b0cb 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -60,7 +60,7 @@ function createDatabase(): PluginDatabaseManager { new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: 'better-sqlite3', connection: ':memory:', }, }, diff --git a/yarn.lock b/yarn.lock index 2527c4985d..66320f0939 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6865,13 +6865,6 @@ "@typescript-eslint/types" "5.9.0" eslint-visitor-keys "^3.0.0" -"@vscode/sqlite3@^5.0.7": - version "5.0.7" - resolved "https://registry.npmjs.org/@vscode/sqlite3/-/sqlite3-5.0.7.tgz#358df36bb0e9e735c54785e3e4b9b2dce1d32895" - integrity sha512-NlsOf+Hir2r4zopI1qMvzWXPwPJuFscirkmFTniTAT24Yz2FWcyZxzK7UT8iSNiTqOCPz48yF55ZVHaz7tTuVQ== - dependencies: - node-addon-api "^4.2.0" - "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" @@ -18250,11 +18243,6 @@ node-abort-controller@^3.0.1: resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== -node-addon-api@^4.2.0: - version "4.3.0" - resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz#52a1a0b475193e0928e98e0426a0d1254782b77f" - integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ== - node-cache@^5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d"