paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

tests: use describe.each for database test iteration

Browse Source 
Refactors all test files that use TestDatabases/TestCaches with
it.each(databases.eachSupportedId()) to instead use describe.each at
the outer level. This ensures that all tests for one database engine
complete before moving to the next, rather than interleaving engines
across individual tests. This reduces the number of concurrent database
connections and should help with test timeout issues in CI.

The TestDatabases.create() call is hoisted to module scope so the
describe.each can iterate over supported IDs at the top level.

40 files changed across packages/backend-defaults,
packages/backend-test-utils, and multiple plugins.

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2026-05-15 17:15:23 +02:00
parent 18aa6e4d56
commit 8165184fba
40 changed files with 4406 additions and 4992 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts
+53 -56
View File
@@ -24,13 +24,13 @@ import { ConfigReader } from '@backstage/config';
jest.setTimeout(60_000);
describe('createPluginKeySource', () => {
const databases = TestDatabases.create();
const mockDir = createMockDirectory();
const databases = TestDatabases.create();
const mockDir = createMockDirectory();
it.each(databases.eachSupportedId())(
'works for implicit database (no config), %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'createPluginKeySource, %p',
databaseId => {
it('works for implicit database (no config)', async () => {
const knex = await databases.init(databaseId);
const getClient = jest.fn(async () => knex);
@@ -61,12 +61,9 @@ describe('createPluginKeySource', () => {
}),
],
});
},
);
});
it.each(databases.eachSupportedId())(
'works for explicit database, %p',
async databaseId => {
it('works for explicit database', async () => {
const knex = await databases.init(databaseId);
const getClient = jest.fn(async () => knex);
@@ -99,11 +96,12 @@ describe('createPluginKeySource', () => {
}),
],
});
},
);
});
},
);
it('works for static', async () => {
const privateKey = `
it('works for static', async () => {
const privateKey = `
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgR8Ja2ppMEgOm1KeY
Kpje00U1luybndt6yC263vcgeKqhRANCAAS+slUrS9JXgtHB1RcDnmlveuu4H3Zm
@@ -111,60 +109,59 @@ describe('createPluginKeySource', () => {
-----END PRIVATE KEY-----
`.trim();
const publicKey = `
const publicKey = `
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvrJVK0vSV4LRwdUXA55pb3rruB92
ZoUEY72HTvjIP9xSbLOhWhMREk84T0q91/m3v3sWq5EzHIWw1zRHQisKZw==
-----END PUBLIC KEY-----
`.trim();
mockDir.setContent({
'public.pem': publicKey,
'private.pem': privateKey,
});
const publicKeyPath = mockDir.resolve('public.pem');
const privateKeyPath = mockDir.resolve('private.pem');
mockDir.setContent({
'public.pem': publicKey,
'private.pem': privateKey,
});
const publicKeyPath = mockDir.resolve('public.pem');
const privateKeyPath = mockDir.resolve('private.pem');
const getClient = jest.fn();
const getClient = jest.fn();
const source = await createPluginKeySource({
config: new ConfigReader({
backend: {
auth: {
pluginKeyStore: {
type: 'static',
static: {
keys: [
{
publicKeyFile: publicKeyPath,
privateKeyFile: privateKeyPath,
keyId: '1',
},
],
},
const source = await createPluginKeySource({
config: new ConfigReader({
backend: {
auth: {
pluginKeyStore: {
type: 'static',
static: {
keys: [
{
publicKeyFile: publicKeyPath,
privateKeyFile: privateKeyPath,
keyId: '1',
},
],
},
},
},
}),
database: mockServices.database.mock({ getClient }),
logger: mockServices.logger.mock(),
keyDuration: { seconds: 10 },
});
},
}),
database: mockServices.database.mock({ getClient }),
logger: mockServices.logger.mock(),
keyDuration: { seconds: 10 },
});
expect(getClient).not.toHaveBeenCalled();
expect(getClient).not.toHaveBeenCalled();
const keys = await source.listKeys();
expect(keys.keys.length).toEqual(1);
expect(keys.keys[0].key).toMatchObject({
kid: '1',
alg: 'ES256',
});
const keys = await source.listKeys();
expect(keys.keys.length).toEqual(1);
expect(keys.keys[0].key).toMatchObject({
kid: '1',
alg: 'ES256',
});
const pk = await source.getPrivateSigningKey();
expect(pk).toMatchObject({
kid: '1',
alg: 'ES256',
d: expect.any(String),
});
const pk = await source.getPrivateSigningKey();
expect(pk).toMatchObject({
kid: '1',
alg: 'ES256',
d: expect.any(String),
});
});
packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts
Vendored
+39 -43
View File
@@ -54,14 +54,16 @@ jest.mock('@keyv/memcache', () => {
};
});
describe('CacheManager integration', () => {
const caches = TestCaches.create();
jest.setTimeout(60_000);
afterEach(jest.clearAllMocks);
const caches = TestCaches.create();
it.each(caches.eachSupportedId())(
'only creates one underlying connection per plugin, %p',
async cacheId => {
describe.each(caches.eachSupportedId())(
'CacheManager integration, %p',
cacheId => {
afterEach(jest.clearAllMocks);
it('only creates one underlying connection per plugin', async () => {
const { store, connection } = await caches.init(cacheId);
const manager = CacheManager.fromConfig(
@@ -85,12 +87,9 @@ describe('CacheManager integration', () => {
// eslint-disable-next-line jest/no-conditional-expect
expect(KeyvValkey).toHaveBeenCalledTimes(3);
}
},
);
});
it.each(caches.eachSupportedId())(
'interacts correctly with store, %p',
async cacheId => {
it('interacts correctly with store', async () => {
const { store, connection } = await caches.init(cacheId);
const manager = CacheManager.fromConfig(
@@ -114,12 +113,9 @@ describe('CacheManager integration', () => {
await expect(plugin1.get('a')).resolves.toBe('plugin1');
await expect(plugin2a.get('a')).resolves.toBe('plugin2b');
await expect(plugin2b.get('a')).resolves.toBe('plugin2b');
},
);
});
it.each(caches.eachSupportedId())(
'supports both milliseconds and human durations throughout, %p',
async cacheId => {
it('supports both milliseconds and human durations throughout', async () => {
const { store, connection } = await caches.init(cacheId);
for (const defaultTtl of [200, { milliseconds: 200 }]) {
@@ -175,39 +171,39 @@ describe('CacheManager integration', () => {
await expect(defaultClient.get('e')).resolves.toBeUndefined();
await expect(defaultClient.get('f')).resolves.toBeUndefined();
}
},
);
});
},
);
it('rejects invalid defaultTtl', () => {
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
},
it('rejects invalid defaultTtl', () => {
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
},
},
}),
),
).not.toThrow();
},
}),
),
).not.toThrow();
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
defaultTtl: 'hello',
},
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
defaultTtl: 'hello',
},
},
}),
),
).toThrow(/Invalid duration 'hello' in config/);
});
},
}),
),
).toThrow(/Invalid duration 'hello' in config/);
});
describe('CacheManager store options', () => {
packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts
+51 -54
View File
@@ -41,68 +41,65 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20250411000000_last_run.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20250411000000_last_run.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250411000000_last_run.js');
await migrateUntilBefore(knex, '20250411000000_last_run.js');
await knex
.insert({
id: 'i',
settings_json: '{}',
})
.into('backstage_backend_tasks__tasks');
await knex
.insert({
id: 'i',
settings_json: '{}',
})
.into('backstage_backend_tasks__tasks');
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateUpOnce(knex);
await migrateUpOnce(knex);
await knex
.table('backstage_backend_tasks__tasks')
.update({ last_run_error_json: 'error' })
.where({ id: 'i' });
await knex
.table('backstage_backend_tasks__tasks')
.update({ last_run_error_json: 'error' })
.where({ id: 'i' });
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
last_run_ended_at: null,
last_run_error_json: 'error',
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
last_run_ended_at: null,
last_run_error_json: 'error',
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await knex.destroy();
},
);
await knex.destroy();
});
});
packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts
+49 -54
View File
@@ -24,9 +24,10 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('TaskScheduler', () => {
const databases = TestDatabases.create();
describe.each(databases.eachSupportedId())('TaskScheduler, %p', databaseId => {
const logger = mockServices.logger.mock();
const databases = TestDatabases.create();
const rootLifecycle = mockServices.rootLifecycle.mock();
const httpRouter = mockServices.httpRouter.mock();
const pluginMetadata = {
@@ -35,63 +36,57 @@ describe('TaskScheduler', () => {
const testScopedSignal = createTestScopedSignal();
const metrics = metricsServiceMock.mock();
it.each(databases.eachSupportedId())(
'can return a working v1 plugin impl, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
it('can return a working v1 plugin impl', async () => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromMillis(5000),
signal: testScopedSignal(),
fn,
});
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromMillis(5000),
signal: testScopedSignal(),
fn,
});
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
},
);
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
});
it.each(databases.eachSupportedId())(
'can return a working v2 plugin impl, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
it('can return a working v2 plugin impl', async () => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
signal: testScopedSignal(),
fn,
});
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
signal: testScopedSignal(),
fn,
});
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
},
);
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
});
});
packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts
+220 -246
View File
@@ -31,48 +31,50 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('PluginTaskManagerImpl', () => {
const addShutdownHook = jest.fn();
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
beforeAll(async () => {
// Make sure all databases are running before mocking timers, in case of testcontainers
await Promise.all(
databases.eachSupportedId().map(([id]) => databases.init(id)),
);
describe.each(databases.eachSupportedId())(
'PluginTaskManagerImpl, %p',
databaseId => {
const addShutdownHook = jest.fn();
jest.useFakeTimers();
}, 60_000);
beforeAll(async () => {
// Make sure the database is running before mocking timers, in case of testcontainers
await databases.init(databaseId);
jest.useFakeTimers();
}, 60_000);
beforeEach(() => {
jest.clearAllMocks();
});
afterAll(() => {
jest.useRealTimers();
});
async function init(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await migrateBackendTasks(knex);
const manager = new PluginTaskSchedulerImpl(
'myplugin',
async () => knex,
mockServices.logger.mock(),
metricsServiceMock.mock(),
{
addShutdownHook,
addBeforeShutdownHook: jest.fn(),
addStartupHook: jest.fn(),
},
);
return { knex, manager };
}
beforeEach(() => {
jest.clearAllMocks();
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can run the v1 happy path, %p',
async databaseId => {
async function init(id: TestDatabaseId) {
const knex = await databases.init(id);
await migrateBackendTasks(knex);
const manager = new PluginTaskSchedulerImpl(
'myplugin',
async () => knex,
mockServices.logger.mock(),
metricsServiceMock.mock(),
{
addShutdownHook,
addBeforeShutdownHook: jest.fn(),
addStartupHook: jest.fn(),
},
);
return { knex, manager };
}
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with global scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -87,12 +89,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'can run the v2 happy path, %p',
async databaseId => {
it('can run the v2 happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -107,12 +106,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'aborts the task if shutdown hook is invoked, %p',
async databaseId => {
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -134,14 +130,11 @@ describe('PluginTaskManagerImpl', () => {
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
},
);
});
});
});
describe('triggerTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can manually trigger a task, %p',
async databaseId => {
describe('triggerTask with global scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -160,12 +153,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'cant trigger a non-existent task, %p',
async databaseId => {
it('cant trigger a non-existent task', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -180,12 +170,9 @@ describe('PluginTaskManagerImpl', () => {
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
},
);
});
it.each(databases.eachSupportedId())(
'cant trigger a running task, %p',
async databaseId => {
it('cant trigger a running task', async () => {
const { manager } = await init(databaseId);
const promise = createDeferred();
@@ -205,140 +192,137 @@ describe('PluginTaskManagerImpl', () => {
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
},
);
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with local scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: { milliseconds: 5000 },
frequency: { milliseconds: 5000 },
fn,
scope: 'local',
});
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with local scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init('SQLITE_3');
it('can run the v2 happy path', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: { milliseconds: 5000 },
frequency: { milliseconds: 5000 },
fn,
scope: 'local',
});
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
it('can run the v2 happy path', async () => {
const { manager } = await init('SQLITE_3');
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
const fn = jest.fn();
const promise = new Promise<AbortSignal>(resolve =>
fn.mockImplementation(resolve),
);
await manager.scheduleTask({
id: 'task3',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
const shutdownHook = addShutdownHook.mock.calls[0][0];
const abortSignal = await promise;
expect(abortSignal.aborted).toBe(false);
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init('SQLITE_3');
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
}, 60_000);
});
const fn = jest.fn();
const promise = new Promise<AbortSignal>(resolve =>
fn.mockImplementation(resolve),
);
await manager.scheduleTask({
id: 'task3',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
describe('triggerTask with local scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init('SQLITE_3');
const shutdownHook = addShutdownHook.mock.calls[0][0];
const abortSignal = await promise;
expect(abortSignal.aborted).toBe(false);
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
}, 60_000);
});
await manager.triggerTask('task1');
jest.advanceTimersByTime(5000);
describe('triggerTask with local scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init('SQLITE_3');
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
it('cant trigger a non-existent task', async () => {
const { manager } = await init('SQLITE_3');
await manager.triggerTask('task1');
jest.advanceTimersByTime(5000);
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
}, 60_000);
it('cant trigger a non-existent task', async () => {
const { manager } = await init('SQLITE_3');
it('cant trigger a running task', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
const promise = createDeferred();
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
}, 60_000);
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
it('cant trigger a running task', async () => {
const { manager } = await init('SQLITE_3');
await promise;
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
const promise = createDeferred();
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('createScheduledTaskRunner', () => {
it.each(databases.eachSupportedId())(
'can run the happy path, %p',
async databaseId => {
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
await promise;
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('createScheduledTaskRunner', () => {
it('can run the happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -356,14 +340,11 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
});
});
describe('can fetch task ids', () => {
it.each(databases.eachSupportedId())(
'can fetch both global and local task ids, %p',
async databaseId => {
describe('can fetch task ids', () => {
it('can fetch both global and local task ids', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -395,52 +376,51 @@ describe('PluginTaskManagerImpl', () => {
settings: expect.objectContaining({ cadence: 'PT5S' }),
},
]);
},
);
});
describe('cancelTask with local scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init('SQLITE_3');
const promise = createDeferred();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
});
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
}, 60_000);
describe('cancelTask with local scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init('SQLITE_3');
it('cannot cancel a task that is not running', async () => {
const { manager } = await init('SQLITE_3');
const promise = createDeferred();
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
await expect(manager.cancelTask('task1')).rejects.toThrow(ConflictError);
}, 60_000);
});
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
}, 60_000);
describe('cancelTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can cancel a running task, %p',
async databaseId => {
it('cannot cancel a task that is not running', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await expect(manager.cancelTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
describe('cancelTask with global scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init(databaseId);
const promise = createDeferred();
@@ -458,23 +438,17 @@ describe('PluginTaskManagerImpl', () => {
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'cannot cancel a non-existent task, %p',
async databaseId => {
it('cannot cancel a non-existent task', async () => {
const { manager } = await init(databaseId);
await expect(manager.cancelTask('nonexistent')).rejects.toThrow(
NotFoundError,
);
},
);
});
it.each(databases.eachSupportedId())(
'cannot cancel a task that is not running, %p',
async databaseId => {
it('cannot cancel a task that is not running', async () => {
const { manager } = await init(databaseId);
await manager.scheduleTask({
@@ -489,16 +463,16 @@ describe('PluginTaskManagerImpl', () => {
await expect(manager.cancelTask('task1')).rejects.toThrow(
ConflictError,
);
},
);
});
describe('parseDuration', () => {
it('should parse durations', () => {
expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S');
expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S');
expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *');
expect(parseDuration({ trigger: 'manual' })).toEqual('manual');
});
});
});
});
describe('parseDuration', () => {
it('should parse durations', () => {
expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S');
expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S');
expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *');
expect(parseDuration({ trigger: 'manual' })).toEqual('manual');
});
});
},
);
packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts
+23 -22
View File
@@ -34,28 +34,29 @@ const getTask = async (knex: Knex): Promise<DbTasksRow> => {
return (await knex<DbTasksRow>(DB_TASKS_TABLE))[0];
};
describe('PluginTaskSchedulerJanitor', () => {
const logger = mockServices.logger.mock();
const databases = TestDatabases.create({
ids: [
/* 'MYSQL_8' not supported yet */
'POSTGRES_18',
'POSTGRES_14',
'SQLITE_3',
'MYSQL_8',
],
});
const testScopedSignal = createTestScopedSignal();
jest.setTimeout(60_000);
jest.setTimeout(60_000);
const databases = TestDatabases.create({
ids: [
/* 'MYSQL_8' not supported yet */
'POSTGRES_18',
'POSTGRES_14',
'SQLITE_3',
'MYSQL_8',
],
});
beforeEach(() => {
jest.resetAllMocks();
});
describe.each(databases.eachSupportedId())(
'PluginTaskSchedulerJanitor, %p',
databaseId => {
const logger = mockServices.logger.mock();
const testScopedSignal = createTestScopedSignal();
it.each(databases.eachSupportedId())(
'Should update date if current_run_expires_at expires, %p',
async databaseId => {
beforeEach(() => {
jest.resetAllMocks();
});
it('Should update date if current_run_expires_at expires', async () => {
const knex = await databases.init(databaseId);
await migrateBackendTasks(knex);
@@ -92,6 +93,6 @@ describe('PluginTaskSchedulerJanitor', () => {
}),
);
});
},
);
});
});
},
);
packages/backend-defaults/src/migrations.test.ts
+57 -63
View File
@@ -41,83 +41,77 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20210928160613_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20210928160613_init.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20210928160613_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20210928160613_init.js');
await migrateUpOnce(knex);
await knex('backstage_backend_tasks__tasks').insert({
await knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.fn.now(),
});
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: knex.fn.now(),
});
next_run_start_at: expect.anything(),
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: expect.anything(),
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual(
expect.anything(),
);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual(
expect.anything(),
);
await knex.destroy();
});
await knex.destroy();
},
);
it('20240712211735_nullable_next_run.js', async () => {
const knex = await databases.init(databaseId);
it.each(databases.eachSupportedId())(
'20240712211735_nullable_next_run.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js');
await migrateUpOnce(knex);
await knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
});
await knex('backstage_backend_tasks__tasks').insert({
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await expect(
knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
});
}),
).rejects.toEqual(expect.anything());
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await expect(
knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
}),
).rejects.toEqual(expect.anything());
await knex.destroy();
},
);
await knex.destroy();
});
});
packages/backend-test-utils/src/cache/TestCaches.test.ts
Vendored
+3 -3
View File
@@ -21,10 +21,10 @@ const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
jest.setTimeout(60_000);
describe('TestCaches', () => {
const caches = TestCaches.create();
const caches = TestCaches.create();
it.each(caches.eachSupportedId())('fires up a cache, %p', async cacheId => {
describe.each(caches.eachSupportedId())('TestCaches, %p', cacheId => {
it('fires up a cache', async () => {
const { keyv } = await caches.init(cacheId);
await keyv.set('test', 'value');
await expect(keyv.get('test')).resolves.toBe('value');
packages/backend-test-utils/src/database/TestDatabases.test.ts
+8 -18
View File
@@ -18,24 +18,14 @@ import { TestDatabases } from './TestDatabases';
jest.setTimeout(60_000);
describe('TestDatabases', () => {
describe('each create', () => {
const dbs = TestDatabases.create();
const dbs = TestDatabases.create();
it.each(dbs.eachSupportedId())(
'creates distinct %p databases',
async databaseId => {
if (!dbs.supports(databaseId)) {
return;
}
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([
{ a: 1 },
]);
},
);
describe.each(dbs.eachSupportedId())('TestDatabases, %p', databaseId => {
it('creates distinct databases', async () => {
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([{ a: 1 }]);
});
});
plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
+14 -23
View File
@@ -19,12 +19,12 @@ import { StaticAssetsStore } from './StaticAssetsStore';
jest.setTimeout(60_000);
describe('StaticAssetsStore', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'should store and retrieve assets, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'StaticAssetsStore, %p',
databaseId => {
it('should store and retrieve assets', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -61,12 +61,9 @@ describe('StaticAssetsStore', () => {
await expect(
store.getAsset('does-not-exist.txt'),
).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'should update assets timestamps, but not contents, %p',
async databaseId => {
it('should update assets timestamps, but not contents', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -112,12 +109,9 @@ describe('StaticAssetsStore', () => {
const sameBar = await store.getAsset('bar');
expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt);
},
);
});
it.each(databases.eachSupportedId())(
'should trim old assets, %p',
async databaseId => {
it('should trim old assets', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -157,12 +151,9 @@ describe('StaticAssetsStore', () => {
await expect(store.getAsset('new')).resolves.toBeDefined();
await expect(store.getAsset('old')).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'should isolate assets in namespace, %p',
async databaseId => {
it('should isolate assets in namespace', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -197,6 +188,6 @@ describe('StaticAssetsStore', () => {
await otherStore.trimAssets({ maxAgeSeconds: 0 });
await expect(otherStore.getAsset('bar')).resolves.not.toBeDefined();
},
);
});
});
},
);
plugins/app-backend/src/migrations.test.ts
+69 -75
View File
@@ -41,101 +41,95 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20211229105307_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20211229105307_init.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20211229105307_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20211229105307_init.js');
await migrateUpOnce(knex);
await knex('static_assets_cache').insert({
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('static_assets_cache')).rejects.toEqual(
expect.anything(),
);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('static_assets_cache')).rejects.toEqual(
expect.anything(),
);
await knex.destroy();
});
await knex.destroy();
},
);
it('20240113144027_assets-namespace.js', async () => {
const knex = await databases.init(databaseId);
it.each(databases.eachSupportedId())(
'20240113144027_assets-namespace.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240113144027_assets-namespace.js');
await migrateUntilBefore(knex, '20240113144027_assets-namespace.js');
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
await knex('static_assets_cache').insert({
await migrateUpOnce(knex);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
namespace: 'default',
last_modified_at: expect.anything(),
},
]);
await migrateUpOnce(knex);
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: knex.fn.now(),
});
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
]);
await knex('static_assets_cache').insert({
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
{
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: knex.fn.now(),
});
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
{
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: expect.anything(),
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await knex.destroy();
},
);
await knex.destroy();
});
});
plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts
+10 -13
View File
@@ -27,12 +27,12 @@ const keyBase = {
jest.setTimeout(60_000);
describe('DatabaseKeyStore', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'should store a key, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'DatabaseKeyStore, %p',
databaseId => {
it('should store a key', async () => {
const knex = await databases.init(databaseId);
await AuthDatabase.runMigrations(knex);
@@ -53,12 +53,9 @@ describe('DatabaseKeyStore', () => {
DateTime.fromJSDate(items[0].createdAt).diffNow('seconds').seconds,
),
).toBeLessThan(10);
},
);
});
it.each(databases.eachSupportedId())(
'should remove stored keys, %p',
async databaseId => {
it('should remove stored keys', async () => {
const knex = await databases.init(databaseId);
await AuthDatabase.runMigrations(knex);
@@ -124,6 +121,6 @@ describe('DatabaseKeyStore', () => {
await expect(store.listKeys()).resolves.toEqual({
items: [],
});
},
);
});
});
},
);
plugins/auth-backend/src/identity/KeyStores.test.ts
+66 -75
View File
@@ -24,89 +24,80 @@ import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
jest.setTimeout(60_000);
describe('KeyStores', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
const defaultConfigOptions = {
auth: {
keyStore: {
provider: 'memory',
},
const defaultConfigOptions = {
auth: {
keyStore: {
provider: 'memory',
},
};
const defaultConfig = new ConfigReader(defaultConfigOptions);
},
};
const defaultConfig = new ConfigReader(defaultConfigOptions);
it.each(databases.eachSupportedId())(
'reads auth section from config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig');
const keyStore = await KeyStores.fromConfig(defaultConfig, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
describe.each(databases.eachSupportedId())('KeyStores, %p', databaseId => {
it('reads auth section from config', async () => {
const knex = await databases.init(databaseId);
const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig');
const keyStore = await KeyStores.fromConfig(defaultConfig, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(MemoryKeyStore);
expect(configSpy).toHaveBeenCalledWith('auth.keyStore');
expect(
defaultConfig
.getOptionalConfig('auth.keyStore')
?.getOptionalString('provider'),
).toBe(defaultConfigOptions.auth.keyStore.provider);
},
);
expect(keyStore).toBeInstanceOf(MemoryKeyStore);
expect(configSpy).toHaveBeenCalledWith('auth.keyStore');
expect(
defaultConfig
.getOptionalConfig('auth.keyStore')
?.getOptionalString('provider'),
).toBe(defaultConfigOptions.auth.keyStore.provider);
});
it.each(databases.eachSupportedId())(
'can handle without auth config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const keyStore = await KeyStores.fromConfig(new ConfigReader({}), {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(DatabaseKeyStore);
},
);
it('can handle without auth config', async () => {
const knex = await databases.init(databaseId);
const keyStore = await KeyStores.fromConfig(new ConfigReader({}), {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(DatabaseKeyStore);
});
it.each(databases.eachSupportedId())(
'can handle additional provider config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue();
const createSpy = jest.spyOn(FirestoreKeyStore, 'create');
it('can handle additional provider config', async () => {
const knex = await databases.init(databaseId);
jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue();
const createSpy = jest.spyOn(FirestoreKeyStore, 'create');
const configOptions = {
auth: {
keyStore: {
provider: 'firestore',
firestore: {
projectId: 'my-project',
keyFilename: 'cred.json',
path: 'my-path',
timeout: 100,
host: 'localhost',
port: 8088,
ssl: false,
},
const configOptions = {
auth: {
keyStore: {
provider: 'firestore',
firestore: {
projectId: 'my-project',
keyFilename: 'cred.json',
path: 'my-path',
timeout: 100,
host: 'localhost',
port: 8088,
ssl: false,
},
},
};
const config = new ConfigReader(configOptions);
const keyStore = await KeyStores.fromConfig(config, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
},
};
const config = new ConfigReader(configOptions);
const keyStore = await KeyStores.fromConfig(config, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
expect(createSpy).toHaveBeenCalledWith(
configOptions.auth.keyStore.firestore,
);
expect(
config
.getOptionalConfig('auth.keyStore')
?.getOptionalConfig('firestore')
?.getOptionalString('projectId'),
).toBe(configOptions.auth.keyStore.firestore.projectId);
},
);
expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
expect(createSpy).toHaveBeenCalledWith(
configOptions.auth.keyStore.firestore,
);
expect(
config
.getOptionalConfig('auth.keyStore')
?.getOptionalConfig('firestore')
?.getOptionalString('projectId'),
).toBe(configOptions.auth.keyStore.firestore.projectId);
});
});
plugins/auth-backend/src/migrations.test.ts
+330 -350
View File
@@ -41,179 +41,154 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20230428155633_sessions.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20230428155633_sessions.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20230428155633_sessions.js');
await migrateUntilBefore(knex, '20230428155633_sessions.js');
await migrateUpOnce(knex);
// Ensure that large cookies are supported
const data = `{"cookie":"${'a'.repeat(100_000)}"}`;
await knex
.insert({ sid: 'abc', expired: knex.fn.now(), sess: data })
.into('sessions');
await knex
.insert({ sid: 'def', expired: knex.fn.now(), sess: data })
.into('sessions');
await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([
{ sid: 'abc', expired: expect.anything(), sess: data },
{ sid: 'def', expired: expect.anything(), sess: data },
]);
await migrateDownOnce(knex);
await knex.destroy();
});
it('20240510120825_user_info.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240510120825_user_info.js');
await migrateUpOnce(knex);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await expect(knex('user_info')).resolves.toEqual([
{
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: expect.anything(),
},
]);
await migrateDownOnce(knex);
await knex.destroy();
});
it('20250707164600_user_created_at.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250707164600_user_created_at.js');
if (knex.client.config.client.includes('sqlite')) {
// Sqlite doesn't support adding a column with non-constant default when table has data
// so we just test that the migration runs without errors
await migrateUpOnce(knex);
// Ensure that large cookies are supported
const data = `{"cookie":"${'a'.repeat(100_000)}"}`;
await knex
.insert({ sid: 'abc', expired: knex.fn.now(), sess: data })
.into('sessions');
await knex
.insert({ sid: 'def', expired: knex.fn.now(), sess: data })
.into('sessions');
return;
}
await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([
{ sid: 'abc', expired: expect.anything(), sess: data },
{ sid: 'def', expired: expect.anything(), sess: data },
]);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await migrateDownOnce(knex);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await knex.destroy();
},
);
const { exp } = await knex('user_info').first();
it.each(databases.eachSupportedId())(
'20240510120825_user_info.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240510120825_user_info.js');
await migrateUpOnce(knex);
const { created_at, updated_at } = await knex('user_info').first();
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
expect(updated_at).toEqual(exp);
expect(created_at).toBeDefined();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info')
.onConflict(['user_entity_ref'])
.merge();
await expect(knex('user_info')).resolves.toEqual([
{
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: expect.anything(),
},
]);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user-2',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info');
await migrateDownOnce(knex);
await expect(
knex('user_info').select('created_at', 'updated_at'),
).resolves.toEqual([
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
]);
await knex.destroy();
},
);
await migrateDownOnce(knex);
it.each(databases.eachSupportedId())(
'20250707164600_user_created_at.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250707164600_user_created_at.js');
await expect(knex('user_info').select('exp')).resolves.toEqual([
{ exp: expect.any(Date) },
{ exp: expect.any(Date) },
]);
if (knex.client.config.client.includes('sqlite')) {
// Sqlite doesn't support adding a column with non-constant default when table has data
// so we just test that the migration runs without errors
await migrateUpOnce(knex);
await knex.destroy();
});
return;
}
it('20250909120000_oidc_client_registration.js', async () => {
const knex = await databases.init(databaseId);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await migrateUntilBefore(
knex,
'20250909120000_oidc_client_registration.js',
);
await migrateUpOnce(knex);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
const { exp } = await knex('user_info').first();
await migrateUpOnce(knex);
const { created_at, updated_at } = await knex('user_info').first();
expect(updated_at).toEqual(exp);
expect(created_at).toBeDefined();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info')
.onConflict(['user_entity_ref'])
.merge();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user-2',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info');
await expect(
knex('user_info').select('created_at', 'updated_at'),
).resolves.toEqual([
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
]);
await migrateDownOnce(knex);
await expect(knex('user_info').select('exp')).resolves.toEqual([
{ exp: expect.any(Date) },
{ exp: expect.any(Date) },
]);
await knex.destroy();
},
);
it.each(databases.eachSupportedId())(
'20250909120000_oidc_client_registration.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(
knex,
'20250909120000_oidc_client_registration.js',
);
await migrateUpOnce(knex);
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
})
.into('oidc_clients');
await expect(
knex('oidc_clients').where('client_id', 'test-client-id').first(),
).resolves.toEqual({
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
@@ -222,230 +197,235 @@ describe('migrations', () => {
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
});
})
.into('oidc_clients');
await knex
.insert({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await expect(
knex('oidc_clients').where('client_id', 'test-client-id').first(),
).resolves.toEqual({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
});
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
}),
);
await knex
.insert({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex
.insert({
code: 'test-auth-code',
session_id: 'test-session-id',
expires_at: new Date(Date.now() + 600000),
used: false,
})
.into('oidc_authorization_codes');
await expect(
knex('oidc_authorization_codes')
.where('code', 'test-auth-code')
.first(),
).resolves.toEqual(
expect.objectContaining({
code: 'test-auth-code',
session_id: 'test-session-id',
}),
);
await knex('oauth_authorization_sessions')
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.del();
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
}),
);
await expect(
knex('oidc_authorization_codes').where('session_id', 'test-session-id'),
).resolves.toHaveLength(0);
await knex
.insert({
code: 'test-auth-code',
session_id: 'test-session-id',
expires_at: new Date(Date.now() + 600000),
used: false,
})
.into('oidc_authorization_codes');
await migrateDownOnce(knex);
await expect(
knex('oidc_authorization_codes').where('code', 'test-auth-code').first(),
).resolves.toEqual(
expect.objectContaining({
code: 'test-auth-code',
session_id: 'test-session-id',
}),
);
const tables = [
'oidc_clients',
'oauth_authorization_sessions',
'oidc_authorization_codes',
];
await knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.del();
for (const table of tables) {
await expect(knex.schema.hasTable(table)).resolves.toBe(false);
}
await expect(
knex('oidc_authorization_codes').where('session_id', 'test-session-id'),
).resolves.toHaveLength(0);
await knex.destroy();
},
);
await migrateDownOnce(knex);
it.each(databases.eachSupportedId())(
'20251118120000_oauth_state_text.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const tables = [
'oidc_clients',
'oauth_authorization_sessions',
'oidc_authorization_codes',
];
await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js');
for (const table of tables) {
await expect(knex.schema.hasTable(table)).resolves.toBe(false);
}
// First create a client for the foreign key constraint
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
await knex.destroy();
});
// Insert a session with state before migration
const existingState = 'existing-short-state';
await knex
.insert({
id: 'test-existing-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: existingState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it('20251118120000_oauth_state_text.js', async () => {
const knex = await databases.init(databaseId);
// Apply the migration that changes state to TEXT
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js');
// Verify existing state persists after migration
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-existing-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-existing-session',
state: existingState,
}),
);
// First create a client for the foreign key constraint
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
// Test inserting a state parameter longer than 255 characters
// This is based on the real-world example from the issue
const longState = 'a'.repeat(280);
// Insert a session with state before migration
const existingState = 'existing-short-state';
await knex
.insert({
id: 'test-existing-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: existingState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex
.insert({
id: 'test-long-state-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: longState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Apply the migration that changes state to TEXT
await migrateUpOnce(knex);
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-long-state-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-long-state-session',
state: longState,
}),
);
// Verify existing state persists after migration
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-existing-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-existing-session',
state: existingState,
}),
);
await migrateDownOnce(knex);
// Test inserting a state parameter longer than 255 characters
// This is based on the real-world example from the issue
const longState = 'a'.repeat(280);
await knex.destroy();
},
);
await knex
.insert({
id: 'test-long-state-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: longState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it.each(databases.eachSupportedId())(
'20251217120000_drop_oidc_clients_fk.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-long-state-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-long-state-session',
state: longState,
}),
);
await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js');
await migrateDownOnce(knex);
// Create a client for DCR sessions
await knex
.insert({
client_id: 'dcr-client-id',
client_secret: 'test-client-secret',
client_name: 'DCR Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
await knex.destroy();
});
// Create a DCR session (has matching client in oidc_clients)
await knex
.insert({
id: 'dcr-session',
client_id: 'dcr-client-id',
redirect_uri: 'https://example.com/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it('20251217120000_drop_oidc_clients_fk.js', async () => {
const knex = await databases.init(databaseId);
// Apply migration - drops FK constraint
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js');
// Now we can insert a CIMD session (URL-based client_id not in oidc_clients)
await knex
.insert({
id: 'cimd-session',
client_id: 'https://example.com/.well-known/oauth-client/cli',
redirect_uri: 'http://localhost:8080/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Create a client for DCR sessions
await knex
.insert({
client_id: 'dcr-client-id',
client_secret: 'test-client-secret',
client_name: 'DCR Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
// Verify both sessions exist
await expect(
knex('oauth_authorization_sessions').select('id').orderBy('id'),
).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]);
// Create a DCR session (has matching client in oidc_clients)
await knex
.insert({
id: 'dcr-session',
client_id: 'dcr-client-id',
redirect_uri: 'https://example.com/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Rollback - should delete CIMD sessions and re-add FK
await migrateDownOnce(knex);
// Apply migration - drops FK constraint
await migrateUpOnce(knex);
// CIMD session should be deleted, DCR session should remain
await expect(
knex('oauth_authorization_sessions').select('id'),
).resolves.toEqual([{ id: 'dcr-session' }]);
// Now we can insert a CIMD session (URL-based client_id not in oidc_clients)
await knex
.insert({
id: 'cimd-session',
client_id: 'https://example.com/.well-known/oauth-client/cli',
redirect_uri: 'http://localhost:8080/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex.destroy();
},
);
// Verify both sessions exist
await expect(
knex('oauth_authorization_sessions').select('id').orderBy('id'),
).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]);
// Rollback - should delete CIMD sessions and re-add FK
await migrateDownOnce(knex);
// CIMD session should be deleted, DCR session should remain
await expect(
knex('oauth_authorization_sessions').select('id'),
).resolves.toEqual([{ id: 'dcr-session' }]);
await knex.destroy();
});
});
plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts
+18 -17
View File
@@ -23,18 +23,20 @@ const migrationsDir = `${__dirname}/../../migrations`;
jest.setTimeout(60_000);
describe('IncrementalIngestionDatabaseManager', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
'stores and retrieves marks, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'IncrementalIngestionDatabaseManager, %p',
databaseId => {
it('stores and retrieves marks', async () => {
const knex = await databases.init(databaseId);
await knex.migrate.latest({ directory: migrationsDir });
const manager = new IncrementalIngestionDatabaseManager({ client: knex });
const manager = new IncrementalIngestionDatabaseManager({
client: knex,
});
const { ingestionId } = (await manager.createProviderIngestionRecord(
'myProvider',
))!;
@@ -75,16 +77,15 @@ describe('IncrementalIngestionDatabaseManager', () => {
sequence: 1,
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'computeRemoved correctly sums total count from count query, %p',
async databaseId => {
it('computeRemoved correctly sums total count from count query', async () => {
const knex = await databases.init(databaseId);
await knex.migrate.latest({ directory: migrationsDir });
const manager = new IncrementalIngestionDatabaseManager({ client: knex });
const manager = new IncrementalIngestionDatabaseManager({
client: knex,
});
const { ingestionId } = (await manager.createProviderIngestionRecord(
'testProvider',
))!;
@@ -119,6 +120,6 @@ describe('IncrementalIngestionDatabaseManager', () => {
// On PostgreSQL, count queries return strings, so total should be 3 not NaN or string concatenation
expect(result.total).toBe(3);
expect(typeof result.total).toBe('number');
},
);
});
});
},
);
plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts
+20 -19
View File
@@ -23,24 +23,25 @@ import { WrapperProviders } from './WrapperProviders';
jest.setTimeout(60_000);
describe('WrapperProviders', () => {
const applyDatabaseMigrations = jest.fn();
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'],
});
const config = new ConfigReader({});
const logger = mockServices.logger.mock();
const scheduler = {
scheduleTask: jest.fn(),
};
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'],
});
beforeEach(() => {
jest.clearAllMocks();
});
describe.each(databases.eachSupportedId())(
'WrapperProviders, %p',
databaseId => {
const applyDatabaseMigrations = jest.fn();
const config = new ConfigReader({});
const logger = mockServices.logger.mock();
const scheduler = {
scheduleTask: jest.fn(),
};
it.each(databases.eachSupportedId())(
'should initialize the providers in order, %p',
async databaseId => {
beforeEach(() => {
jest.clearAllMocks();
});
it('should initialize the providers in order', async () => {
const client = await databases.init(databaseId);
const provider1: IncrementalEntityProvider<number, {}> = {
@@ -111,6 +112,6 @@ describe('WrapperProviders', () => {
id: 'provider2',
}),
);
},
);
});
});
},
);
plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts
+38 -44
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { DefaultCatalogDatabase } from './DefaultCatalogDatabase';
import { applyDatabaseMigrations } from './migrations';
import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';
@@ -26,48 +22,46 @@ import { LoggerService } from '@backstage/backend-plugin-api';
jest.setTimeout(60_000);
describe('DefaultCatalogDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
describe.each(databases.eachSupportedId())(
'DefaultCatalogDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
describe('listAncestors', () => {
let nextId = 1;
function makeEntity(ref: string) {
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
knex,
db: new DefaultCatalogDatabase({
database: knex,
logger,
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return ancestors, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('listAncestors', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it('should return ancestors', async () => {
const { knex, db } = await createDatabase();
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
@@ -107,7 +101,7 @@ describe('DefaultCatalogDatabase', () => {
'location:default/root-1',
'location:default/root-2',
]);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts
+110 -152
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
@@ -40,64 +36,62 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('DefaultProcessingDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: createRandomProcessingInterval({
minSeconds: 100,
maxSeconds: 150,
describe.each(databases.eachSupportedId())(
'DefaultProcessingDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: createRandomProcessingInterval({
minSeconds: 100,
maxSeconds: 150,
}),
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
};
}
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
describe('updateProcessedEntity', () => {
let id: string;
let processedEntity: Entity;
beforeEach(() => {
id = uuid();
processedEntity = {
apiVersion: '1',
kind: 'Location',
metadata: {
name: 'fakelocation',
},
spec: {
type: 'url',
target: 'somethingelse',
},
};
});
}
it.each(databases.eachSupportedId())(
'fails when an entity is processed with a different locationKey, %p',
async databaseId => {
const { db } = await createDatabase(databaseId);
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
describe('updateProcessedEntity', () => {
let id: string;
let processedEntity: Entity;
beforeEach(() => {
id = uuid();
processedEntity = {
apiVersion: '1',
kind: 'Location',
metadata: {
name: 'fakelocation',
},
spec: {
type: 'url',
target: 'somethingelse',
},
};
});
it('fails when an entity is processed with a different locationKey', async () => {
const { db } = await createDatabase();
await db.transaction(async tx => {
await expect(() =>
db.updateProcessedEntity(tx, {
@@ -112,12 +106,9 @@ describe('DefaultProcessingDatabase', () => {
`Conflicting write of processing result for ${id} with location key 'undefined'`,
);
});
},
);
});
it.each(databases.eachSupportedId())(
'fails when the locationKey is different, %p',
async databaseId => {
it('fails when the locationKey is different', async () => {
const options = {
id,
processedEntity,
@@ -128,7 +119,7 @@ describe('DefaultProcessingDatabase', () => {
refreshKeys: [],
errors: "['something broke']",
};
const { knex, db } = await createDatabase(databaseId);
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -158,13 +149,10 @@ describe('DefaultProcessingDatabase', () => {
`Conflicting write of processing result for ${id} with location key 'fail'`,
),
);
},
);
});
it.each(databases.eachSupportedId())(
'updates the refresh state entry with the cache, processed entity and errors, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('updates the refresh state entry with the cache, processed entity and errors', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -197,13 +185,10 @@ describe('DefaultProcessingDatabase', () => {
);
expect(entities[0].errors).toEqual("['something broke']");
expect(entities[0].location_key).toEqual('key');
},
);
});
it.each(databases.eachSupportedId())(
'removes old relations and stores the new relationships, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('removes old relations and stores the new relationships', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -282,13 +267,10 @@ describe('DefaultProcessingDatabase', () => {
target_entity_ref: 'component:default/foo',
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'adds deferred entities to the refresh_state table to be picked up later, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('adds deferred entities to the refresh_state table to be picked up later', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -330,19 +312,15 @@ describe('DefaultProcessingDatabase', () => {
.select();
expect(refreshStateEntries).toHaveLength(1);
},
);
});
it.each(databases.eachSupportedId())(
'updates unprocessed entities with varying location keys, %p',
async databaseId => {
it('updates unprocessed entities with varying location keys', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
@@ -479,19 +457,15 @@ describe('DefaultProcessingDatabase', () => {
expect(mockLogger.error).not.toHaveBeenCalled();
}
});
},
);
});
it.each(databases.eachSupportedId())(
'stores the refresh keys for the entity where key length is 255 chars or less',
async databaseId => {
it('stores the refresh keys for the entity where key length is 255 chars or less', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
await insertRefreshStateRow(knex, {
@@ -536,19 +510,15 @@ describe('DefaultProcessingDatabase', () => {
entity_id: id,
key: 'protocol:foo-bar.com',
});
},
);
});
it.each(databases.eachSupportedId())(
'stores the refresh keys for the entity where key length is greater than 255 chars',
async databaseId => {
it('stores the refresh keys for the entity where key length is greater than 255 chars', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
await insertRefreshStateRow(knex, {
@@ -597,15 +567,12 @@ describe('DefaultProcessingDatabase', () => {
entity_id: id,
key: `url:https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-back#sha256:edfb606500d184900e63891e5279d35bf0069ea251e90d15c0a430de6023d905`,
});
},
);
});
});
});
describe('updateEntityCache', () => {
it.each(databases.eachSupportedId())(
'updates the entityCache, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('updateEntityCache', () => {
it('updates the entityCache', async () => {
const { knex, db } = await createDatabase();
const id = '123';
await insertRefreshStateRow(knex, {
entity_id: id,
@@ -644,15 +611,12 @@ describe('DefaultProcessingDatabase', () => {
).select();
expect(entities2.length).toBe(1);
expect(entities2[0].cache).toEqual('{}');
},
);
});
});
});
describe('getProcessableEntities', () => {
it.each(databases.eachSupportedId())(
'should return entities to process, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('getProcessableEntities', () => {
it('should return entities to process', async () => {
const { knex, db } = await createDatabase();
const entity = JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
@@ -696,13 +660,10 @@ describe('DefaultProcessingDatabase', () => {
}),
).resolves.toEqual({ items: [] });
});
},
);
});
it.each(databases.eachSupportedId())(
'should update the next_refresh interval with a timestamp that includes refresh spread, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should update the next_refresh interval with a timestamp that includes refresh spread', async () => {
const { knex, db } = await createDatabase();
const entity = JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
@@ -731,33 +692,30 @@ describe('DefaultProcessingDatabase', () => {
const nextUpdate = timestampToDateTime(result[0].next_update_at);
const nextUpdateDiff = nextUpdate.diff(now, 'seconds');
expect(nextUpdateDiff.seconds).toBeGreaterThanOrEqual(90);
},
);
});
});
});
describe('listParents', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
describe('listParents', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return parents, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should return parents', async () => {
const { knex, db } = await createDatabase();
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
@@ -803,7 +761,7 @@ describe('DefaultProcessingDatabase', () => {
db.listParents(tx, { entityRefs: ['location:default/root-2'] }),
);
expect(result3.entityRefs).toEqual([]);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts
+88 -133
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
@@ -30,54 +26,52 @@ import { generateStableHash } from './util';
jest.setTimeout(60_000);
describe('DefaultProviderDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProviderDatabase({
database: knex,
logger,
}),
};
}
describe.each(databases.eachSupportedId())(
'DefaultProviderDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProviderDatabase({
database: knex,
logger,
}),
};
}
};
describe('replaceUnprocessedEntities', () => {
it.each(databases.eachSupportedId())(
'replaces all existing state correctly for simple dependency chains, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
};
describe('replaceUnprocessedEntities', () => {
it('replaces all existing state correctly for simple dependency chains', async () => {
const { knex, db } = await createDatabase();
/*
config -> location:default/root -> location:default/root-1 -> location:default/root-2
database -> location:default/second -> location:default/root-2
@@ -187,13 +181,10 @@ describe('DefaultProviderDatabase', () => {
t.source_key === 'config',
),
).toBeTruthy();
},
);
});
it.each(databases.eachSupportedId())(
'should work for more complex chains, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should work for more complex chains', async () => {
const { knex, db } = await createDatabase();
/*
config -> location:default/root -> location:default/root-1 -> location:default/root-2
config -> location:default/root -> location:default/root-1a -> location:default/root-2
@@ -323,13 +314,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/root-2',
),
).toBeFalsy();
},
);
});
it.each(databases.eachSupportedId())(
'should add new locations using the delta options, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should add new locations using the delta options', async () => {
const { knex, db } = await createDatabase();
// Existing state and references should stay
await createLocations(knex, ['location:default/existing']);
@@ -393,13 +381,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/existing',
),
).toBeTruthy();
},
);
});
it.each(databases.eachSupportedId())(
'should not remove locations that are referenced elsewhere, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should not remove locations that are referenced elsewhere', async () => {
const { knex, db } = await createDatabase();
/*
config-1 -> location:default/root
config-2 -> location:default/root
@@ -443,13 +428,10 @@ describe('DefaultProviderDatabase', () => {
entity_ref: 'location:default/root',
}),
]);
},
);
});
it.each(databases.eachSupportedId())(
'should remove old locations using the delta options, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should remove old locations using the delta options', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, ['location:default/new-root']);
await insertRefRow(knex, {
@@ -492,13 +474,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/new-root',
),
).toBeFalsy();
},
);
});
it.each(databases.eachSupportedId())(
'should update the location key during full replace, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should update the location key during full replace', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, ['location:default/removed']);
await insertRefreshStateRow(knex, {
entity_id: uuid(),
@@ -558,13 +537,10 @@ describe('DefaultProviderDatabase', () => {
target_entity_ref: 'location:default/replaced',
}),
]);
},
);
});
it.each(databases.eachSupportedId())(
'should support replacing modified entities during a full update, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should support replacing modified entities during a full update', async () => {
const { knex, db } = await createDatabase();
await db.transaction(async tx => {
await db.replaceUnprocessedEntities(tx, {
@@ -689,14 +665,11 @@ describe('DefaultProviderDatabase', () => {
target_entity_ref: 'component:default/a',
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'should successfully fall back from batch to individual mode on conflicts, %p',
async databaseId => {
it('should successfully fall back from batch to individual mode on conflicts', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
await createLocations(knex, ['component:default/a']);
@@ -738,14 +711,11 @@ describe('DefaultProviderDatabase', () => {
}),
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync, %p',
async databaseId => {
it('should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
await createLocations(knex, ['component:default/a']);
@@ -768,14 +738,11 @@ describe('DefaultProviderDatabase', () => {
const state = await knex<DbRefreshStateRow>('refresh_state').select();
expect(state).toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'should properly translate deltas into add/update/remove, %p',
async databaseId => {
it('should properly translate deltas into add/update/remove', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
const entity1Before: Entity = {
apiVersion: '1',
@@ -950,14 +917,11 @@ describe('DefaultProviderDatabase', () => {
location_key: 'new', // managed to update only the location key
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'can handle large deltas without exploding, %p',
async databaseId => {
it('can handle large deltas without exploding', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
const count = 10000;
const padded = (n: number) => String(n).padStart(8, '0');
@@ -989,15 +953,12 @@ describe('DefaultProviderDatabase', () => {
unprocessed_entity: JSON.stringify(entities[0].entity),
unprocessed_hash: generateStableHash(entities[0].entity),
});
},
);
});
});
});
describe('listReferenceSourceKeys', () => {
it.each(databases.eachSupportedId())(
'returns the source_keys from "refresh_state_references", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('listReferenceSourceKeys', () => {
it('returns the source_keys from "refresh_state_references"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1018,13 +979,10 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['bar', 'foo']);
},
);
});
it.each(databases.eachSupportedId())(
'returns only unique source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('returns only unique source_keys"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1045,13 +1003,10 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['foo']);
},
);
});
it.each(databases.eachSupportedId())(
'does not return null source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('does not return null source_keys"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1071,7 +1026,7 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['foo']);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/metrics.test.ts
+84 -93
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from './migrations';
@@ -23,10 +23,10 @@ import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics';
jest.setTimeout(60_000);
describe('metrics', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
describe.each(databases.eachSupportedId())('metrics, %p', databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
@@ -55,108 +55,99 @@ describe('metrics', () => {
}
describe('queryEntitiesCountByKind', () => {
it.each(databases.eachSupportedId())(
'counts entities grouped by the kind in entity_ref, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
it('counts entities grouped by the kind in entity_ref', async () => {
const knex = await createDatabase();
await insertEntity(knex, {
entityRef: 'component:default/svc-a',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'component:default/svc-b',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'api:default/api-a',
finalEntity: '{"kind":"API"}',
});
await insertEntity(knex, {
entityRef: 'system:other/sys-a',
finalEntity: '{"kind":"System"}',
});
// Not yet stitched -- must be excluded from the count
await insertEntity(knex, {
entityRef: 'component:default/pending',
finalEntity: null,
});
await insertEntity(knex, {
entityRef: 'component:default/svc-a',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'component:default/svc-b',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'api:default/api-a',
finalEntity: '{"kind":"API"}',
});
await insertEntity(knex, {
entityRef: 'system:other/sys-a',
finalEntity: '{"kind":"System"}',
});
// Not yet stitched -- must be excluded from the count
await insertEntity(knex, {
entityRef: 'component:default/pending',
finalEntity: null,
});
const result = await queryEntitiesCountByKind(knex);
const result = await queryEntitiesCountByKind(knex);
expect(Object.fromEntries(result)).toEqual({
component: 2,
api: 1,
system: 1,
});
},
);
expect(Object.fromEntries(result)).toEqual({
component: 2,
api: 1,
system: 1,
});
});
});
describe('createEntitiesCountByKind', () => {
it.each(databases.eachSupportedId())(
'serves cached results within the TTL and refreshes after, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
it('serves cached results within the TTL and refreshes after', async () => {
const knex = await createDatabase();
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
const first = await getCount();
expect(Object.fromEntries(first)).toEqual({ component: 1 });
const first = await getCount();
expect(Object.fromEntries(first)).toEqual({ component: 1 });
// A change made within the TTL window must not be visible yet.
await insertEntity(knex, {
entityRef: 'component:default/two',
finalEntity: '{}',
});
const cached = await getCount();
expect(Object.fromEntries(cached)).toEqual({ component: 1 });
// A change made within the TTL window must not be visible yet.
await insertEntity(knex, {
entityRef: 'component:default/two',
finalEntity: '{}',
});
const cached = await getCount();
expect(Object.fromEntries(cached)).toEqual({ component: 1 });
// After the TTL elapses the next call hits the database again.
await new Promise(resolve => setTimeout(resolve, 80));
const refreshed = await getCount();
expect(Object.fromEntries(refreshed)).toEqual({ component: 2 });
},
);
// After the TTL elapses the next call hits the database again.
await new Promise(resolve => setTimeout(resolve, 80));
const refreshed = await getCount();
expect(Object.fromEntries(refreshed)).toEqual({ component: 2 });
});
it.each(databases.eachSupportedId())(
'coalesces overlapping callers into a single underlying query, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
it('coalesces overlapping callers into a single underlying query', async () => {
const knex = await createDatabase();
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
const finalEntitiesQueries: string[] = [];
knex.on('query', (q: { sql: string }) => {
if (
/from\s+["`]?final_entities["`]?/i.test(q.sql) &&
/^\s*select/i.test(q.sql)
) {
finalEntitiesQueries.push(q.sql);
}
});
// Five concurrent callers should result in one query, not five.
const results = await Promise.all([
getCount(),
getCount(),
getCount(),
getCount(),
getCount(),
]);
expect(finalEntitiesQueries).toHaveLength(1);
for (const r of results) {
expect(Object.fromEntries(r)).toEqual({ component: 1 });
const finalEntitiesQueries: string[] = [];
knex.on('query', (q: { sql: string }) => {
if (
/from\s+["`]?final_entities["`]?/i.test(q.sql) &&
/^\s*select/i.test(q.sql)
) {
finalEntitiesQueries.push(q.sql);
}
},
);
});
// Five concurrent callers should result in one query, not five.
const results = await Promise.all([
getCount(),
getCount(),
getCount(),
getCount(),
getCount(),
]);
expect(finalEntitiesQueries).toHaveLength(1);
for (const r of results) {
expect(Object.fromEntries(r)).toEqual({ component: 1 });
}
});
});
});
plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts
+79 -94
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from '../../migrations';
@@ -27,72 +27,72 @@ import { deleteWithEagerPruningOfChildren } from './deleteWithEagerPruningOfChil
jest.setTimeout(60_000);
describe('deleteWithEagerPruningOfChildren', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
return knex<DbRefreshStateReferencesRow>('refresh_state_references').insert(
refs,
);
}
async function insertRelation(
knex: Knex,
...relations: { from: string; to: string }[]
) {
for (const rel of relations) {
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: await knex<DbRefreshStateRow>('refresh_state')
.select('entity_id')
.then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion
source_entity_ref: rel.from,
target_entity_ref: rel.to,
type: 'fake',
});
describe.each(databases.eachSupportedId())(
'deleteWithEagerPruningOfChildren, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
return knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert(refs);
}
}
async function remainingEntities(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref');
return rows.map(r => r.entity_ref);
}
async function insertRelation(
knex: Knex,
...relations: { from: string; to: string }[]
) {
for (const rel of relations) {
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: await knex<DbRefreshStateRow>('refresh_state')
.select('entity_id')
.then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion
source_entity_ref: rel.from,
target_entity_ref: rel.to,
type: 'fake',
});
}
}
async function entitiesMarkedForStitching(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref')
.where('result_hash', '=', 'force-stitching');
return rows.map(r => r.entity_ref);
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
}
it.each(databases.eachSupportedId())(
'works for the simple path, %p',
async databaseId => {
async function remainingEntities(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref');
return rows.map(r => r.entity_ref);
}
async function entitiesMarkedForStitching(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref')
.where('result_hash', '=', 'force-stitching');
return rows.map(r => r.entity_ref);
}
it('works for the simple path', async () => {
/*
P1 - E1 - E2
@@ -106,7 +106,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1, E2, and E3 deleted; E4 and E5 remain; E4 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5');
await insertReference(
knex,
@@ -124,12 +124,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E4', 'E5']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E4']);
},
);
});
it.each(databases.eachSupportedId())(
'works when there are multiple identical references, %p',
async databaseId => {
it('works when there are multiple identical references', async () => {
/*
P1
\
@@ -143,7 +140,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 remains; E2 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2');
await insertReference(
knex,
@@ -159,12 +156,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']);
},
);
});
it.each(databases.eachSupportedId())(
'leaves out things that have roots in other source keys, %p',
async databaseId => {
it('leaves out things that have roots in other source keys', async () => {
/*
P1 - E1
\
@@ -176,7 +170,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 and E3 remain; E2 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3');
await insertReference(
knex,
@@ -197,12 +191,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']);
},
);
});
it.each(databases.eachSupportedId())(
'leaves out things that have several different roots for the same source key, %p',
async databaseId => {
it('leaves out things that have several different roots for the same source key', async () => {
/*
P1 - E1
\
@@ -214,7 +205,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 and E3 remain
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3');
await insertReference(
knex,
@@ -230,12 +221,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'handles cycles and diamonds gracefully, %p',
async databaseId => {
it('handles cycles and diamonds gracefully', async () => {
/*
P1 - E1 <-> E2
\
@@ -247,7 +235,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: Everything deleted, but in two steps; E4 marked for stitching in the first step because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5', 'E6');
await insertReference(
knex,
@@ -281,12 +269,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual([]);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'silently ignores attempts to delete things that are not your own and/or are not roots, %p',
async databaseId => {
it('silently ignores attempts to delete things that are not your own and/or are not roots', async () => {
/*
P1 - E1 - E2
@@ -298,7 +283,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E3 is deleted; E1, E2 and E4 remain
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4');
await insertReference(
knex,
@@ -318,6 +303,6 @@ describe('deleteWithEagerPruningOfChildren', () => {
'E4',
]);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts
+15 -15
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from '../../migrations';
import { DbRefreshKeysRow, DbRefreshStateRow } from '../../tables';
@@ -23,19 +23,19 @@ import { refreshByRefreshKeys } from './refreshByRefreshKeys';
jest.setTimeout(60_000);
describe('refreshByRefreshKeys', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
describe.each(databases.eachSupportedId())(
'refreshByRefreshKeys, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
it.each(databases.eachSupportedId())(
'works for the simple path, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
it('works for the simple path', async () => {
const knex = await createDatabase();
const eid1 = uuid();
await knex<DbRefreshStateRow>('refresh_state').insert({
@@ -89,6 +89,6 @@ describe('refreshByRefreshKeys', () => {
expect(normalizeTimestamp(before2.next_update_at)).toEqual(
normalizeTimestamp(after2.next_update_at),
);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts
+8 -8
View File
@@ -21,12 +21,12 @@ import { getDeferredStitchableEntities } from './getDeferredStitchableEntities';
jest.setTimeout(60_000);
describe('getDeferredStitchableEntities', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'selects the right rows %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'getDeferredStitchableEntities, %p',
databaseId => {
it('selects the right rows', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -83,6 +83,6 @@ describe('getDeferredStitchableEntities', () => {
expect(+new Date(hitRowAfter!)).toBeGreaterThan(+new Date(hitRowBefore!));
expect(+new Date(missRowAfter!)).toEqual(+new Date(missRowBefore!));
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts
+8 -8
View File
@@ -21,12 +21,12 @@ import { DbStitchQueueRow } from '../../tables';
jest.setTimeout(60_000);
describe('markDeferredStitchCompleted', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'completes only if unchanged %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'markDeferredStitchCompleted, %p',
databaseId => {
it('completes only if unchanged', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -68,6 +68,6 @@ describe('markDeferredStitchCompleted', () => {
stitchTicket: 'the-ticket',
});
await expect(result()).resolves.toEqual([]);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts
+12 -18
View File
@@ -25,12 +25,12 @@ import {
jest.setTimeout(60_000);
describe('markForStitching', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'marks the right rows in deferred mode %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'markForStitching, %p',
databaseId => {
it('marks the right rows in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -203,12 +203,9 @@ describe('markForStitching', () => {
const final = await result();
const entity4Final = final.find(r => r.entity_ref === 'k:ns/four');
expect(entity4Final?.stitch_ticket).not.toEqual('old');
},
);
});
it.each(databases.eachSupportedId())(
'marks the right rows in immediate mode %p',
async databaseId => {
it('marks the right rows in immediate mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -435,12 +432,9 @@ describe('markForStitching', () => {
for (let i = 0; i < final.length; ++i) {
expect(original[i].next_update_at).not.toEqual(final[i].next_update_at);
}
},
);
});
it.each(databases.eachSupportedId())(
'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p',
async databaseId => {
it('reproduces deadlock scenario when concurrent transactions update overlapping entity sets', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -567,6 +561,6 @@ describe('markForStitching', () => {
expect(row.next_stitch_at).not.toBeNull();
expect(row.stitch_ticket).not.toBeNull();
});
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts
+15 -20
View File
@@ -29,14 +29,15 @@ import { performStitching } from './performStitching';
jest.setTimeout(60_000);
describe('performStitching', () => {
const databases = TestDatabases.create();
const logger = mockServices.logger.mock();
const databases = TestDatabases.create();
// NOTE(freben): Testing the deferred path since it's a superset of the immediate one
it.each(databases.eachSupportedId())(
'runs the happy path in deferred mode for %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'performStitching, %p',
databaseId => {
const logger = mockServices.logger.mock();
// NOTE(freben): Testing the deferred path since it's a superset of the immediate one
it('runs the happy path in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -323,12 +324,9 @@ describe('performStitching', () => {
},
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'handles conflicts with past stitches %p',
async databaseId => {
it('handles conflicts with past stitches', async () => {
if (databaseId === 'MYSQL_8') {
// MySQL doesn't handle conflicts in the same way as the other two, most
// likely due to the conflict probably being handled with a merged even
@@ -391,12 +389,9 @@ describe('performStitching', () => {
'Skipping stitching of k:ns/n, conflict',
expect.anything(),
);
},
);
});
it.each(databases.eachSupportedId())(
'stitches when final_entities row already exists %p',
async databaseId => {
it('stitches when final_entities row already exists', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -444,6 +439,6 @@ describe('performStitching', () => {
expect(entities.length).toBe(1);
expect(entities[0].hash).not.toBe('');
expect(entities[0].final_entity).toBeDefined();
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts
+105 -105
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { StitchingStrategy } from '../../../stitching/types';
import { applyDatabaseMigrations } from '../../migrations';
@@ -28,109 +28,112 @@ import { deleteOrphanedEntities } from './deleteOrphanedEntities';
jest.setTimeout(60_000);
describe('deleteOrphanedEntities', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
describe.each(databases.eachSupportedId())(
'deleteOrphanedEntities, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
async function run(knex: Knex, strategy: StitchingStrategy): Promise<number> {
let result: number;
await knex.transaction(
async tx => {
// We can't return here, as knex swallows the return type in case the
// transaction is rolled back:
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
result = await deleteOrphanedEntities({ knex: tx, strategy });
},
{
// If we explicitly trigger a rollback, don't fail.
doNotRejectOnRollback: true,
},
);
return result!;
}
async function run(
knex: Knex,
strategy: StitchingStrategy,
): Promise<number> {
let result: number;
await knex.transaction(
async tx => {
// We can't return here, as knex swallows the return type in case the
// transaction is rolled back:
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
result = await deleteOrphanedEntities({ knex: tx, strategy });
},
{
// If we explicitly trigger a rollback, don't fail.
doNotRejectOnRollback: true,
},
);
return result!;
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: `id-${ref}`,
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
result_hash: 'original',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: `id-${ref}`,
hash: 'original',
entity_ref: ref,
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: `id-${ref}`,
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
result_hash: 'original',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: `id-${ref}`,
hash: 'original',
entity_ref: ref,
});
}
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert(refs);
}
async function insertRelation(knex: Knex, fromRef: string, toRef: string) {
const orig = await knex
.select('entity_id')
.from('refresh_state')
.where('entity_ref', fromRef);
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: orig[0].entity_id,
type: 'fake',
source_entity_ref: fromRef,
target_entity_ref: toRef,
});
}
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
await knex<DbRefreshStateReferencesRow>('refresh_state_references').insert(
refs,
);
}
async function refreshState(knex: Knex) {
return await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref', 'result_hash');
}
async function insertRelation(knex: Knex, fromRef: string, toRef: string) {
const orig = await knex
.select('entity_id')
.from('refresh_state')
.where('entity_ref', fromRef);
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: orig[0].entity_id,
type: 'fake',
source_entity_ref: fromRef,
target_entity_ref: toRef,
});
}
async function stitchQueue(knex: Knex) {
return await knex('stitch_queue')
.orderBy('entity_ref')
.select('entity_ref');
}
async function refreshState(knex: Knex) {
return await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref', 'result_hash');
}
async function finalEntities(knex: Knex) {
return await knex<DbFinalEntitiesRow>('final_entities')
.join(
'refresh_state',
'final_entities.entity_id',
'refresh_state.entity_id',
)
.leftOuterJoin(
'stitch_queue',
'stitch_queue.entity_ref',
'refresh_state.entity_ref',
)
.orderBy('refresh_state.entity_ref')
.select({
entity_ref: 'refresh_state.entity_ref',
hash: 'final_entities.hash',
next_stitch_at: 'stitch_queue.next_stitch_at',
});
}
async function stitchQueue(knex: Knex) {
return await knex('stitch_queue')
.orderBy('entity_ref')
.select('entity_ref');
}
async function finalEntities(knex: Knex) {
return await knex<DbFinalEntitiesRow>('final_entities')
.join(
'refresh_state',
'final_entities.entity_id',
'refresh_state.entity_id',
)
.leftOuterJoin(
'stitch_queue',
'stitch_queue.entity_ref',
'refresh_state.entity_ref',
)
.orderBy('refresh_state.entity_ref')
.select({
entity_ref: 'refresh_state.entity_ref',
hash: 'final_entities.hash',
next_stitch_at: 'stitch_queue.next_stitch_at',
});
}
it.each(databases.eachSupportedId())(
'works for some mixed paths in immediate mode, %p',
async databaseId => {
it('works for some mixed paths in immediate mode', async () => {
/*
In this graph, edges represent refresh state references, not entity relations:
@@ -155,7 +158,7 @@ describe('deleteOrphanedEntities', () => {
Result: E3, E4, E5, E6, and E10 deleted; others remain
Entities that had relations pointing at orphans are marked for reprocessing
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(
knex,
'E1',
@@ -210,12 +213,9 @@ describe('deleteOrphanedEntities', () => {
{ entity_ref: 'E8', hash: 'original', next_stitch_at: null },
{ entity_ref: 'E9', hash: 'original', next_stitch_at: null },
]);
},
);
});
it.each(databases.eachSupportedId())(
'works for some mixed paths in deferred mode, %p',
async databaseId => {
it('works for some mixed paths in deferred mode', async () => {
/*
In this graph, edges represent refresh state references, not entity relations:
@@ -240,7 +240,7 @@ describe('deleteOrphanedEntities', () => {
Result: E3, E4, E5, E6, and E10 deleted; others remain
Entities that had relations pointing at orphans are marked for reprocessing
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(
knex,
'E1',
@@ -304,6 +304,6 @@ describe('deleteOrphanedEntities', () => {
{ entity_ref: 'E8', hash: 'original', next_stitch_at: null },
{ entity_ref: 'E9', hash: 'original', next_stitch_at: null },
]);
},
);
});
});
},
);
plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts
+123 -171
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import {
ANNOTATION_ORIGIN_LOCATION,
stringifyEntityRef,
@@ -37,63 +37,58 @@ import waitFor from 'wait-for-expect';
jest.setTimeout(60_000);
describe('DefaultLocationStore', () => {
const databases = TestDatabases.create();
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
let subscriber: CatalogScmEventsServiceSubscriber | undefined;
const databases = TestDatabases.create();
beforeEach(() => {
jest.clearAllMocks();
describe.each(databases.eachSupportedId())(
'DefaultLocationStore, %p',
databaseId => {
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
let subscriber: CatalogScmEventsServiceSubscriber | undefined;
subscriber = undefined;
mockScmEvents.subscribe.mockImplementation(sub => {
subscriber = sub;
return { unsubscribe: () => {} };
beforeEach(() => {
jest.clearAllMocks();
subscriber = undefined;
mockScmEvents.subscribe.mockImplementation(sub => {
subscriber = sub;
return { unsubscribe: () => {} };
});
});
});
async function createLocationStore(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: true,
unregister: true,
move: true,
});
await store.connect(connection);
return { store, connection, knex };
}
async function createLocationStore() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: true,
unregister: true,
move: true,
});
await store.connect(connection);
return { store, connection, knex };
}
it.each(databases.eachSupportedId())(
'should do a full sync with the locations on connect, %p',
async databaseId => {
const { connection } = await createLocationStore(databaseId);
it('should do a full sync with the locations on connect', async () => {
const { connection } = await createLocationStore();
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
},
);
});
describe('listLocations', () => {
it.each(databases.eachSupportedId())(
'lists empty locations when there is no locations, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('listLocations', () => {
it('lists empty locations when there is no locations', async () => {
const { store } = await createLocationStore();
expect(await store.listLocations()).toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'lists locations that are added to the db, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
it('lists locations that are added to the db', async () => {
const { store } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -111,15 +106,12 @@ describe('DefaultLocationStore', () => {
}),
]),
);
},
);
});
});
});
describe('createLocation', () => {
it.each(databases.eachSupportedId())(
'throws when the location already exists, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('createLocation', () => {
it('throws when the location already exists', async () => {
const { store } = await createLocationStore();
const spec = {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -129,13 +121,10 @@ describe('DefaultLocationStore', () => {
await expect(() => store.createLocation(spec)).rejects.toThrow(
new RegExp(`Location ${spec.type}:${spec.target} already exists`),
);
},
);
});
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -159,13 +148,10 @@ describe('DefaultLocationStore', () => {
},
]),
});
},
);
});
it.each(databases.eachSupportedId())(
'updates refresh_state when onConflict is refresh, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('updates refresh_state when onConflict is refresh', async () => {
const { store, knex } = await createLocationStore();
const spec = {
type: 'url',
target:
@@ -201,13 +187,10 @@ describe('DefaultLocationStore', () => {
expect(new Date(row.next_update_at).getTime()).toBeGreaterThan(
oldDate.getTime(),
);
},
);
});
it.each(databases.eachSupportedId())(
'persists the correct location_entity_ref when creating a location, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('persists the correct location_entity_ref when creating a location', async () => {
const { store, knex } = await createLocationStore();
const created = await store.createLocation({
type: 'url',
target:
@@ -222,26 +205,20 @@ describe('DefaultLocationStore', () => {
expect(row.location_entity_ref).toBe(
'location:default/generated-fa35d9c166e43ab7f4a7c59a00e88e4e8b5aba34',
);
},
);
});
});
});
describe('deleteLocation', () => {
it.each(databases.eachSupportedId())(
'throws if the location does not exist, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('deleteLocation', () => {
it('throws if the location does not exist', async () => {
const { store } = await createLocationStore();
const id = uuid();
await expect(() => store.deleteLocation(id)).rejects.toThrow(
new RegExp(`Found no location with ID ${id}`),
);
},
);
});
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
const location = await store.createLocation({
target:
@@ -268,15 +245,12 @@ describe('DefaultLocationStore', () => {
},
],
});
},
);
});
});
});
describe('updateLocation', () => {
it.each(databases.eachSupportedId())(
'throws if the location does not exist, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('updateLocation', () => {
it('throws if the location does not exist', async () => {
const { store } = await createLocationStore();
const id = uuid();
await expect(() =>
store.updateLocation(id, {
@@ -284,13 +258,10 @@ describe('DefaultLocationStore', () => {
target: 'https://example.com',
}),
).rejects.toThrow(new RegExp(`Found no location with ID ${id}`));
},
);
});
it.each(databases.eachSupportedId())(
'throws ConflictError when updating to a type+target already used by another location, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
it('throws ConflictError when updating to a type+target already used by another location', async () => {
const { store } = await createLocationStore();
await store.createLocation({
type: 'url',
@@ -307,13 +278,10 @@ describe('DefaultLocationStore', () => {
target: 'https://example.com/a',
}),
).rejects.toThrow(/already exists/);
},
);
});
it.each(databases.eachSupportedId())(
'updates type and target and issues a delta mutation with the new entity, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('updates type and target and issues a delta mutation with the new entity', async () => {
const { store, connection } = await createLocationStore();
const created = await store.createLocation({
type: 'url',
@@ -345,15 +313,12 @@ describe('DefaultLocationStore', () => {
},
],
});
},
);
});
});
});
describe('getLocationByEntity', () => {
it.each(databases.eachSupportedId())(
'loads correctly, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
describe('getLocationByEntity', () => {
it('loads correctly', async () => {
const { store, knex } = await createLocationStore();
const entityId = uuid();
const locationId = uuid();
@@ -407,16 +372,12 @@ describe('DefaultLocationStore', () => {
).rejects.toMatchInlineSnapshot(
`[NotFoundError: found no entity for ref k:ns/n2]`,
);
},
);
});
});
});
describe('SCM event handling', () => {
describe.each(databases.eachSupportedId())('%p', databaseId => {
describe('SCM event handling', () => {
it('handles location.deleted', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -531,9 +492,7 @@ describe('DefaultLocationStore', () => {
});
it('handles location.moved', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -669,9 +628,7 @@ describe('DefaultLocationStore', () => {
});
it('handles repository.deleted', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -787,9 +744,7 @@ describe('DefaultLocationStore', () => {
});
it('handles repository.moved', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -919,45 +874,42 @@ describe('DefaultLocationStore', () => {
});
});
});
});
describe('queryLocations', () => {
const l1 = {
id: '00000000-0000-0000-0000-000000000001',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml',
entityRef:
'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a',
};
const l2 = {
id: '00000000-0000-0000-0000-000000000002',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml',
entityRef:
'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce',
};
const l3 = {
id: '00000000-0000-0000-0000-000000000003',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml',
entityRef:
'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e',
};
const l4 = {
id: '00000000-0000-0000-0000-000000000004',
type: 'file',
target: '/tmp/catalog-info.yaml',
entityRef:
'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2',
};
describe('queryLocations', () => {
const l1 = {
id: '00000000-0000-0000-0000-000000000001',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml',
entityRef:
'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a',
};
const l2 = {
id: '00000000-0000-0000-0000-000000000002',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml',
entityRef:
'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce',
};
const l3 = {
id: '00000000-0000-0000-0000-000000000003',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml',
entityRef:
'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e',
};
const l4 = {
id: '00000000-0000-0000-0000-000000000004',
type: 'file',
target: '/tmp/catalog-info.yaml',
entityRef:
'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2',
};
it.each(databases.eachSupportedId())(
'queries locations correctly, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('queries locations correctly', async () => {
const { store, knex } = await createLocationStore();
// Insert locations in a random order to test the sorting
const locations = [l1, l2, l3, l4];
@@ -1195,7 +1147,7 @@ describe('DefaultLocationStore', () => {
items: [],
totalItems: 0,
});
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+236 -365
View File
@@ -15,7 +15,6 @@
*/
import {
TestDatabaseId,
TestDatabases,
mockCredentials,
mockServices,
@@ -42,102 +41,103 @@ import { entitiesResponseToObjects } from './response';
jest.setTimeout(60_000);
describe('DefaultEntitiesCatalog', () => {
let knex: Knex;
const databases = TestDatabases.create();
afterEach(async () => {
await knex.destroy();
});
describe.each(databases.eachSupportedId())(
'DefaultEntitiesCatalog, %p',
databaseId => {
let knex: Knex;
const databases = TestDatabases.create();
const stitch = jest.fn();
const stitcher: Stitcher = { stitch } as any;
async function createDatabase(databaseId: TestDatabaseId) {
knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
}
async function addEntity(
entity: Entity,
parents: { source?: string; entity?: Entity }[],
) {
const id = uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
afterEach(async () => {
await knex.destroy();
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
const stitch = jest.fn();
const stitcher: Stitcher = { stitch } as any;
for (const parent of parents) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: parent.source,
source_entity_ref: parent.entity && stringifyEntityRef(parent.entity),
target_entity_ref: stringifyEntityRef(entity),
});
async function createDatabase() {
knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
}
const search = await buildEntitySearch(id, entity);
await knex<DbSearchRow>('search').insert(search);
async function addEntity(
entity: Entity,
parents: { source?: string; entity?: Entity }[],
) {
const id = uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
return id;
}
async function addEntityToSearch(entity: Entity) {
const id = entity.metadata.uid || uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const row of buildEntitySearch(id, entity)) {
await knex<DbSearchRow>('search').insert({
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
key: row.key,
value: row.value,
original_value: row.original_value,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const parent of parents) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: parent.source,
source_entity_ref: parent.entity && stringifyEntityRef(parent.entity),
target_entity_ref: stringifyEntityRef(entity),
});
}
const search = await buildEntitySearch(id, entity);
await knex<DbSearchRow>('search').insert(search);
return id;
}
}
afterEach(() => {
jest.resetAllMocks();
});
async function addEntityToSearch(entity: Entity) {
const id = entity.metadata.uid || uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
describe('entityAncestry', () => {
it.each(databases.eachSupportedId())(
'should return the ancestry with one parent, %p',
async databaseId => {
await createDatabase(databaseId);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const row of buildEntitySearch(id, entity)) {
await knex<DbSearchRow>('search').insert({
entity_id: id,
key: row.key,
value: row.value,
original_value: row.original_value,
});
}
}
afterEach(() => {
jest.resetAllMocks();
});
describe('entityAncestry', () => {
it('should return the ancestry with one parent', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -188,13 +188,10 @@ describe('DefaultEntitiesCatalog', () => {
},
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'should throw error if the entity does not exist, %p',
async databaseId => {
await createDatabase(databaseId);
it('should throw error if the entity does not exist', async () => {
await createDatabase();
const catalog = new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
@@ -203,13 +200,10 @@ describe('DefaultEntitiesCatalog', () => {
await expect(() =>
catalog.entityAncestry('k:default/root'),
).rejects.toThrow('No such entity k:default/root');
},
);
});
it.each(databases.eachSupportedId())(
'should return the ancestry with multiple parents, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return the ancestry with multiple parents', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -275,15 +269,12 @@ describe('DefaultEntitiesCatalog', () => {
},
]),
);
},
);
});
});
});
describe('entities', () => {
it.each(databases.eachSupportedId())(
'should return correct entity for simple filter, %p',
async databaseId => {
await createDatabase(databaseId);
describe('entities', () => {
it('should return correct entity for simple filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -317,13 +308,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity2);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entity for negation filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entity for negation filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -359,13 +347,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity1);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entities for nested filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entities for nested filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -433,13 +418,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(2);
expect(entities).toContainEqual(entity2);
expect(entities).toContainEqual(entity4);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entities for complex negation filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entities for complex negation filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -480,14 +462,11 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities).toContainEqual(entity1);
},
);
});
it.each(databases.eachSupportedId())(
'should return no matches for an empty values array, %p',
// NOTE: An empty values array is not a sensible input in a realistic scenario.
async databaseId => {
await createDatabase(databaseId);
it('should return no matches for an empty values array', async () => {
// NOTE: An empty values array is not a sensible input in a realistic scenario.
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -519,13 +498,10 @@ describe('DefaultEntitiesCatalog', () => {
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(0);
},
);
});
it.each(databases.eachSupportedId())(
'should return both target and targetRef for entities in compat mode',
async databaseId => {
await createDatabase(databaseId);
it('should return both target and targetRef for entities in compat mode', async () => {
await createDatabase();
await addEntity(
{
apiVersion: 'a',
@@ -579,13 +555,10 @@ describe('DefaultEntitiesCatalog', () => {
target: { kind: 'x', namespace: 'y', name: 'z' },
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'handles inversion both for existing and missing keys, %p',
async databaseId => {
await createDatabase(databaseId);
it('handles inversion both for existing and missing keys', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
@@ -638,13 +611,10 @@ describe('DefaultEntitiesCatalog', () => {
filter: { not: { key: 'spec.b', values: ['lonely'] } },
}),
).resolves.toEqual(['n1', 'n3']);
},
);
});
it.each(databases.eachSupportedId())(
'can order and combine with filtering, %p',
async databaseId => {
await createDatabase(databaseId);
it('can order and combine with filtering', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
@@ -739,13 +709,10 @@ describe('DefaultEntitiesCatalog', () => {
],
}),
).resolves.toEqual(['n4', 'n3', 'n1', 'n2']);
},
);
});
it.each(databases.eachSupportedId())(
'paginates correctly through single-field ordering, %p',
async databaseId => {
await createDatabase(databaseId);
it('paginates correctly through single-field ordering', async () => {
await createDatabase();
// All four entities have metadata.name — fast path uses Phase 1 only
for (const name of ['n1', 'n2', 'n3', 'n4']) {
@@ -795,13 +762,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(await hasNext(2, 1)).toBe(true);
await expect(page(100)).resolves.toEqual(['n1', 'n2', 'n3', 'n4']);
},
);
});
it.each(databases.eachSupportedId())(
'paginates across the Phase 1 / Phase 2 boundary, %p',
async databaseId => {
await createDatabase(databaseId);
it('paginates across the Phase 1 / Phase 2 boundary', async () => {
await createDatabase();
// n1 and n2 have spec.b (Phase 1); n3 and n4 do not (Phase 2).
// Explicit UIDs pin Phase 2 ordering (entity_id ASC) to a known sequence.
@@ -867,13 +831,10 @@ describe('DefaultEntitiesCatalog', () => {
'n3',
'n4',
]);
},
);
});
it.each(databases.eachSupportedId())(
'treats a null sort-field value the same as a missing sort field, %p',
async databaseId => {
await createDatabase(databaseId);
it('treats a null sort-field value the same as a missing sort field', async () => {
await createDatabase();
// n1 has spec.b with a real value (Phase 1)
// n2 has spec.b explicitly set to null — buildEntitySearch stores value=NULL
@@ -918,15 +879,12 @@ describe('DefaultEntitiesCatalog', () => {
// ordered by entity_id ASC, regardless of primary direction
await expect(page('asc')).resolves.toEqual(['n1', 'n2', 'n3']);
await expect(page('desc')).resolves.toEqual(['n1', 'n2', 'n3']);
},
);
});
});
});
describe('entitiesBatch', () => {
it.each(databases.eachSupportedId())(
'queries for entities by ref, including duplicates, and gracefully returns null for missing entities, %p',
async databaseId => {
await createDatabase(databaseId);
describe('entitiesBatch', () => {
it('queries for entities by ref, including duplicates, and gracefully returns null for missing entities', async () => {
await createDatabase();
await addEntity(
{
@@ -976,13 +934,10 @@ describe('DefaultEntitiesCatalog', () => {
null,
'k:default/two',
]);
},
);
});
it.each(databases.eachSupportedId())(
'queries for entities by ref, including filtering, %p',
async databaseId => {
await createDatabase(databaseId);
it('queries for entities by ref, including filtering', async () => {
await createDatabase();
await addEntity(
{
@@ -1022,15 +977,12 @@ describe('DefaultEntitiesCatalog', () => {
'k:default/two',
null,
]);
},
);
});
});
});
describe('queryEntities', () => {
it.each(databases.eachSupportedId())(
'should return paginated entities and scroll the items accordingly, %p',
async databaseId => {
await createDatabase(databaseId);
describe('queryEntities', () => {
it('should return paginated entities and scroll the items accordingly', async () => {
await createDatabase();
const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1201,13 +1153,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
},
);
});
it.each(databases.eachSupportedId())(
'should return paginated entities ordered in descending order and scroll the items accordingly, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return paginated entities ordered in descending order and scroll the items accordingly', async () => {
await createDatabase();
const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1379,13 +1328,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the results when query is provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the results when query is provided', async () => {
await createDatabase();
const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1435,13 +1381,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response.pageInfo.nextCursor).toBeUndefined();
expect(response.pageInfo.prevCursor).toBeUndefined();
expect(response.totalItems).toBe(3);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the results when query is provided with fullTextFilter for camelCase fields, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the results when query is provided with fullTextFilter for camelCase fields', async () => {
await createDatabase();
const entities: Entity[] = [
{
@@ -1489,13 +1432,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response.pageInfo.nextCursor).toBeUndefined();
expect(response.pageInfo.prevCursor).toBeUndefined();
expect(response.totalItems).toBe(1);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the text results when sortOrder is not provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the text results when sortOrder is not provided', async () => {
await createDatabase();
const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
const entities: Entity[] = names.map((name, index) =>
@@ -1577,13 +1517,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the text results by multiple search fields if provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the text results by multiple search fields if provided', async () => {
await createDatabase();
const defs = [
{
@@ -1682,13 +1619,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
},
);
});
it.each(databases.eachSupportedId())(
'should include totalItems and empty entities in the response in case limit is zero, %p',
async databaseId => {
await createDatabase(databaseId);
it('should include totalItems and empty entities in the response in case limit is zero', async () => {
await createDatabase();
await Promise.all(
Array(20)
@@ -1718,13 +1652,10 @@ describe('DefaultEntitiesCatalog', () => {
items: { type: 'raw', entities: [] },
pageInfo: {},
});
},
);
});
it.each(databases.eachSupportedId())(
'can skip totalItems, %p',
async databaseId => {
await createDatabase(databaseId);
it('can skip totalItems', async () => {
await createDatabase();
await Promise.all(
Array(15)
@@ -1772,13 +1703,10 @@ describe('DefaultEntitiesCatalog', () => {
pageInfo: { prevCursor: expect.anything() },
});
expect(response.items.entities).toHaveLength(5);
},
);
});
it.each(databases.eachSupportedId())(
'should paginate results accordingly in case of clashing items, %p',
async databaseId => {
await createDatabase(databaseId);
it('should paginate results accordingly in case of clashing items', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA')),
@@ -1871,13 +1799,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should exclude filtered entities when paginating, %p',
async databaseId => {
await createDatabase(databaseId);
it('should exclude filtered entities when paginating', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: '1', kind: 'included' })),
@@ -1954,13 +1879,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response2.pageInfo.nextCursor).toBeDefined();
expect(response2.pageInfo.prevCursor).toBeDefined();
expect(response2.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should paginate results without sort fields, %p',
async databaseId => {
await createDatabase(databaseId);
it('should paginate results without sort fields', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: 'id1' })),
@@ -2058,13 +1980,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should sort properly for fields that do not exist on all entities, %p',
async databaseId => {
await createDatabase(databaseId);
it('should sort properly for fields that do not exist on all entities', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: 'id1' })),
@@ -2100,13 +2019,10 @@ describe('DefaultEntitiesCatalog', () => {
),
).toEqual(['BB', 'CC']);
expect(descResult.totalItems).toBe(2);
},
);
});
it.each(databases.eachSupportedId())(
'should silently skip over entities that are not yet stitched, %p',
async databaseId => {
await createDatabase(databaseId);
it('should silently skip over entities that are not yet stitched', async () => {
await createDatabase();
const entity1 = entityFrom('AA', { uid: 'id1' });
const entity2 = entityFrom('BB', { uid: 'id2' });
@@ -2149,13 +2065,10 @@ describe('DefaultEntitiesCatalog', () => {
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB']);
},
);
});
it.each(databases.eachSupportedId())(
'should not return duplicate entities when using orderField, %p',
async databaseId => {
await createDatabase(databaseId);
it('should not return duplicate entities when using orderField', async () => {
await createDatabase();
// Create a few test entities with different names to sort by
const entities = [
@@ -2234,13 +2147,10 @@ describe('DefaultEntitiesCatalog', () => {
'b-entity',
'c-entity',
]);
},
);
});
it.each(databases.eachSupportedId())(
'should apply both filter and query when both are given, %p',
async databaseId => {
await createDatabase(databaseId);
it('should apply both filter and query when both are given', async () => {
await createDatabase();
// Add entities with different kinds and names
await addEntityToSearch(entityFrom('A', { kind: 'component' }));
@@ -2266,15 +2176,12 @@ describe('DefaultEntitiesCatalog', () => {
expect(resultEntities).toEqual([
entityFrom('A', { kind: 'component' }),
]);
},
);
});
});
});
describe('removeEntityByUid', () => {
it.each(databases.eachSupportedId())(
'also clears parent hashes, %p',
async databaseId => {
await createDatabase(databaseId);
describe('removeEntityByUid', () => {
it('also clears parent hashes', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -2358,15 +2265,12 @@ describe('DefaultEntitiesCatalog', () => {
expect(stitch).toHaveBeenCalledWith({
entityRefs: new Set(['k:default/unrelated1', 'k:default/unrelated2']),
});
},
);
});
});
});
describe('facets', () => {
it.each(databases.eachSupportedId())(
'can filter and collect properly, %p',
async databaseId => {
await createDatabase(databaseId);
describe('facets', () => {
it('can filter and collect properly', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2432,13 +2336,10 @@ describe('DefaultEntitiesCatalog', () => {
kind: [{ value: 'k', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'can match on annotations and labels with dots in them, %p',
async databaseId => {
await createDatabase(databaseId);
it('can match on annotations and labels with dots in them', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2483,13 +2384,10 @@ describe('DefaultEntitiesCatalog', () => {
],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'can match on strings in arrays, %p',
async databaseId => {
await createDatabase(databaseId);
it('can match on strings in arrays', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2529,13 +2427,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'works with a mixture of present and missing facets, %p',
async databaseId => {
await createDatabase(databaseId);
it('works with a mixture of present and missing facets', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2573,13 +2468,10 @@ describe('DefaultEntitiesCatalog', () => {
missing: [],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'works when the entity is duplicated in search results, %p',
async databaseId => {
await createDatabase(databaseId);
it('works when the entity is duplicated in search results', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2621,28 +2513,22 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'one', count: 1 }],
},
});
},
);
async function setupFacetsCatalog(
databaseId: TestDatabaseId,
entities: Entity[],
) {
await createDatabase(databaseId);
for (const entity of entities) {
await addEntityToSearch(entity);
}
return new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
stitcher,
});
}
it.each(databases.eachSupportedId())(
'excludes not-yet-stitched entities from filtered facets, %p',
async databaseId => {
await createDatabase(databaseId);
async function setupFacetsCatalog(entities: Entity[]) {
await createDatabase();
for (const entity of entities) {
await addEntityToSearch(entity);
}
return new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
stitcher,
});
}
it('excludes not-yet-stitched entities from filtered facets', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2701,13 +2587,10 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'stitched', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with a predicate query, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with a predicate query', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2742,13 +2625,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with a predicate query using $in, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with a predicate query using $in', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2783,13 +2663,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with compound allOf filter, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with compound allOf filter', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2826,13 +2703,10 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'one', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with compound anyOf filter, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with compound anyOf filter', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2872,13 +2746,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with both filter and query combined, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with both filter and query combined', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2911,10 +2782,10 @@ describe('DefaultEntitiesCatalog', () => {
'spec.type': [{ value: 'service', count: 1 }],
},
});
},
);
});
});
});
});
},
);
function entityFrom(
name: string,
plugins/catalog-backend/src/service/DefaultRefreshService.test.ts
+152 -167
View File
@@ -17,7 +17,6 @@
import {
mockCredentials,
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
@@ -42,166 +41,162 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('DefaultRefreshService', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
processingDb: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: () => 100,
events: mockServices.events.mock(),
describe.each(databases.eachSupportedId())(
'DefaultRefreshService, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
processingDb: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: () => 100,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
catalogDb: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
const createPopulatedEngine = async (options: {
db: ProcessingDatabase;
knex: Knex;
entities: Entity[];
references: { [source: string]: string[] };
entityProcessor?: (entity: Entity) => void;
}) => {
const { db, knex, entities, references, entityProcessor } = options;
const entityMap = new Map(
entities.map(entity => [stringifyEntityRef(entity), entity]),
);
for (const entity of entities) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: stringifyEntityRef(entity),
unprocessed_entity: JSON.stringify(entity),
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
const entitiesWithParent = new Set(Object.values(references).flat());
for (const entityRef of entityMap.keys()) {
if (!entitiesWithParent.has(entityRef)) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: 'ConfigLocationProvider',
target_entity_ref: entityRef,
});
}
}
for (const [sourceRef, targetRefs] of Object.entries(references)) {
for (const targetRef of targetRefs) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_entity_ref: sourceRef,
target_entity_ref: targetRef,
});
}
}
const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), {
knex,
logger: defaultLogger,
metrics: metricsServiceMock.mock(),
}),
catalogDb: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
const createPopulatedEngine = async (options: {
db: ProcessingDatabase;
knex: Knex;
entities: Entity[];
references: { [source: string]: string[] };
entityProcessor?: (entity: Entity) => void;
}) => {
const { db, knex, entities, references, entityProcessor } = options;
const entityMap = new Map(
entities.map(entity => [stringifyEntityRef(entity), entity]),
);
for (const entity of entities) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: stringifyEntityRef(entity),
unprocessed_entity: JSON.stringify(entity),
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
const engine = new DefaultCatalogProcessingEngine({
config: new ConfigReader({}),
logger: defaultLogger,
processingDatabase: db,
knex: knex,
stitcher: stitcher,
scheduler: mockServices.scheduler(),
orchestrator: {
async process(request: EntityProcessingRequest) {
const entityRef = stringifyEntityRef(request.entity);
const entity = entityMap.get(entityRef);
if (!entity) {
throw new Error(`Unexpected entity: ${entityRef}`);
}
const deferredEntities =
references[entityRef]?.map(ref => {
const e = entityMap.get(ref);
if (!e) {
throw new Error(`Target entity not found: ${ref}`);
}
return { entity: e, locationKey: ref };
}) || [];
const entitiesWithParent = new Set(Object.values(references).flat());
for (const entityRef of entityMap.keys()) {
if (!entitiesWithParent.has(entityRef)) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: 'ConfigLocationProvider',
target_entity_ref: entityRef,
});
}
}
for (const [sourceRef, targetRefs] of Object.entries(references)) {
for (const targetRef of targetRefs) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_entity_ref: sourceRef,
target_entity_ref: targetRef,
});
}
}
entityProcessor?.(entity);
const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), {
knex,
logger: defaultLogger,
metrics: metricsServiceMock.mock(),
});
const engine = new DefaultCatalogProcessingEngine({
config: new ConfigReader({}),
logger: defaultLogger,
processingDatabase: db,
knex: knex,
stitcher: stitcher,
scheduler: mockServices.scheduler(),
orchestrator: {
async process(request: EntityProcessingRequest) {
const entityRef = stringifyEntityRef(request.entity);
const entity = entityMap.get(entityRef);
if (!entity) {
throw new Error(`Unexpected entity: ${entityRef}`);
}
const deferredEntities =
references[entityRef]?.map(ref => {
const e = entityMap.get(ref);
if (!e) {
throw new Error(`Target entity not found: ${ref}`);
}
return { entity: e, locationKey: ref };
}) || [];
entityProcessor?.(entity);
return {
ok: true,
completedEntity: {
...entity,
metadata: {
...entity.metadata,
annotations: {
...entity.metadata.annotations,
'refresh-completed': 'true',
return {
ok: true,
completedEntity: {
...entity,
metadata: {
...entity.metadata,
annotations: {
...entity.metadata.annotations,
'refresh-completed': 'true',
},
},
},
},
relations: [],
errors: [],
deferredEntities,
state: {},
refreshKeys: [],
};
relations: [],
errors: [],
deferredEntities,
state: {},
refreshKeys: [],
};
},
},
},
createHash: () => createHash('sha1'),
pollingIntervalMs: 50,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
});
createHash: () => createHash('sha1'),
pollingIntervalMs: 50,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
});
return engine;
};
return engine;
};
const waitForRefresh = async (knex: Knex, entityRef: string) => {
for (;;) {
const [result] = await knex<DbRefreshStateRow>('refresh_state')
.where('entity_ref', entityRef)
.select();
const waitForRefresh = async (knex: Knex, entityRef: string) => {
for (;;) {
const [result] = await knex<DbRefreshStateRow>('refresh_state')
.where('entity_ref', entityRef)
.select();
const entity = result.processed_entity
? (JSON.parse(result.processed_entity) as Entity)
: undefined;
if (entity?.metadata?.annotations?.['refresh-completed']) {
// Reset the annotation so that we can run another verification
delete entity.metadata.annotations['refresh-completed'];
await knex<DbRefreshStateRow>('refresh_state')
.update({
processed_entity: JSON.stringify(entity),
})
.where('entity_ref', entityRef);
return true;
const entity = result.processed_entity
? (JSON.parse(result.processed_entity) as Entity)
: undefined;
if (entity?.metadata?.annotations?.['refresh-completed']) {
// Reset the annotation so that we can run another verification
delete entity.metadata.annotations['refresh-completed'];
await knex<DbRefreshStateRow>('refresh_state')
.update({
processed_entity: JSON.stringify(entity),
})
.where('entity_ref', entityRef);
return true;
}
await new Promise(resolve => setTimeout(resolve, 500));
}
await new Promise(resolve => setTimeout(resolve, 500));
}
};
};
it.each(databases.eachSupportedId())(
'should refresh the parent location, %p',
async databaseId => {
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
it('should refresh the parent location', async () => {
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -239,15 +234,10 @@ describe('DefaultRefreshService', () => {
).resolves.toBe(true);
await engine.stop();
},
);
});
it.each(databases.eachSupportedId())(
'should refresh the location further up the tree, %p',
async databaseId => {
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
it('should refresh the location further up the tree', async () => {
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -293,16 +283,11 @@ describe('DefaultRefreshService', () => {
);
await engine.stop();
},
);
});
it.each(databases.eachSupportedId())(
'should refresh even when parent has no changes',
async databaseId => {
it('should refresh even when parent has no changes', async () => {
let secondRound = false;
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -356,6 +341,6 @@ describe('DefaultRefreshService', () => {
).resolves.toBe(true);
await engine.stop();
},
);
});
});
},
);
plugins/catalog-backend/src/service/createRouter.test.ts
+51 -55
View File
@@ -19,7 +19,6 @@ import { wrapServer } from '@backstage/backend-openapi-utils/testUtils';
import {
mockCredentials,
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import type { Location } from '@backstage/catalog-client';
@@ -1680,59 +1679,59 @@ describe('createRouter readonly and raw json enabled', () => {
});
});
describe('POST /locations/by-query works end to end', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createTestRouter(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
describe.each(databases.eachSupportedId())(
'POST /locations/by-query works end to end, %p',
databaseId => {
async function createTestRouter() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: false,
unregister: false,
move: false,
});
await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() });
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: false,
unregister: false,
move: false,
});
await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() });
const locationService = new DefaultLocationService(
store,
{ process: jest.fn() },
{
allowedLocationTypes: ['url'],
defaultLocationConflictStrategy: 'reject',
},
);
const locationService = new DefaultLocationService(
store,
{ process: jest.fn() },
{
allowedLocationTypes: ['url'],
defaultLocationConflictStrategy: 'reject',
},
);
const router = await createRouter({
locationService,
logger: mockServices.logger.mock(),
config: new ConfigReader(undefined),
auth: mockServices.auth(),
httpAuth: mockServices.httpAuth(),
orchestrator: { process: jest.fn() },
permissionsService: mockServices.permissions(),
auditor: mockServices.auditor.mock(),
});
const router = await createRouter({
locationService,
logger: mockServices.logger.mock(),
config: new ConfigReader(undefined),
auth: mockServices.auth(),
httpAuth: mockServices.httpAuth(),
orchestrator: { process: jest.fn() },
permissionsService: mockServices.permissions(),
auditor: mockServices.auditor.mock(),
});
const errorMiddleware = MiddlewareFactory.create({
logger: mockServices.logger.mock(),
config: mockServices.rootConfig(),
});
router.use(errorMiddleware.error());
const errorMiddleware = MiddlewareFactory.create({
logger: mockServices.logger.mock(),
config: mockServices.rootConfig(),
});
router.use(errorMiddleware.error());
return { knex, app: express().use(router) };
}
return { knex, app: express().use(router) };
}
it.each(databases.eachSupportedId())(
'paginates through locations correctly, %p',
async databaseId => {
const { knex, app } = await createTestRouter(databaseId);
it('paginates through locations correctly', async () => {
const { knex, app } = await createTestRouter();
// Insert 5 locations directly into the database
const locations = [
@@ -1819,13 +1818,10 @@ describe('POST /locations/by-query works end to end', () => {
totalItems: 5,
pageInfo: {},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters locations with query parameter, %p',
async databaseId => {
const { knex, app } = await createTestRouter(databaseId);
it('filters locations with query parameter', async () => {
const { knex, app } = await createTestRouter();
// Insert locations with different types
const locations = [
@@ -1874,9 +1870,9 @@ describe('POST /locations/by-query works end to end', () => {
totalItems: 2,
pageInfo: {},
});
},
);
});
});
},
);
function mockCursor(partialCursor?: Partial<Cursor>): Cursor {
return {
plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts
+226 -228
View File
@@ -29,248 +29,246 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('Stitcher', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
describe.each(databases.eachSupportedId())('Stitcher, %p', databaseId => {
const logger = mockServices.logger.mock();
it.each(databases.eachSupportedId())(
'runs the happy path for %p',
async databaseId => {
const db = await databases.init(databaseId);
await applyDatabaseMigrations(db);
it('runs the happy path', async () => {
const db = await databases.init(databaseId);
await applyDatabaseMigrations(db);
const stitcher = new DefaultStitcher({
knex: db,
logger,
strategy: { mode: 'immediate' },
metrics: metricsServiceMock.mock(),
});
let entities: DbFinalEntitiesRow[];
let entity: Entity;
const stitcher = new DefaultStitcher({
knex: db,
logger,
strategy: { mode: 'immediate' },
metrics: metricsServiceMock.mock(),
});
let entities: DbFinalEntitiesRow[];
let entity: Entity;
await db<DbRefreshStateRow>('refresh_state').insert([
await db<DbRefreshStateRow>('refresh_state').insert([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
},
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
// handles and ignores duplicates
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const last_updated_at = entities[0].last_updated_at;
expect(last_updated_at).not.toBeNull();
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
// handles and ignores duplicates
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const last_updated_at = entities[0].last_updated_at;
expect(last_updated_at).not.toBeNull();
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
// Re-stitch without any changes
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
// Re-stitch without any changes
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/third',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
target_entity_ref: 'k:ns/third',
targetRef: 'k:ns/other',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
{
type: 'looksAt',
targetRef: 'k:ns/third',
},
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
{
type: 'looksAt',
targetRef: 'k:ns/third',
},
spec: {
k: 'v',
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/third',
value: 'k:ns/third',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
},
);
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/third',
value: 'k:ns/third',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
});
});
plugins/catalog-backend/src/tests/migrations.test.ts
+1055 -1105
View File
File diff suppressed because it is too large Load Diff
plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts
+10 -14
View File
@@ -144,14 +144,13 @@ class Tracker {
}
}
describePerformanceTest('stitchingPerformance', () => {
const databases = TestDatabases.create({
ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
'runs stitching in immediate mode, %p',
async databaseId => {
describePerformanceTest('stitchingPerformance', () => {
describe.each(databases.eachSupportedId())('%p', databaseId => {
it('runs stitching in immediate mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -199,12 +198,9 @@ describePerformanceTest('stitchingPerformance', () => {
await expect(tracker.completion()).resolves.toBeUndefined();
await backend.stop();
await knex.destroy();
},
);
});
it.each(databases.eachSupportedId())(
'runs stitching in deferred mode, %p',
async databaseId => {
it('runs stitching in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -252,6 +248,6 @@ describePerformanceTest('stitchingPerformance', () => {
await expect(tracker.completion()).resolves.toBeUndefined();
await backend.stop();
await knex.destroy();
},
);
});
});
});
plugins/events-backend/src/migrations.test.ts
+37 -43
View File
@@ -45,60 +45,54 @@ const databases = TestDatabases.create({
ids: ['POSTGRES_9', 'POSTGRES_14', 'POSTGRES_16'],
});
const maybeDescribe =
databases.eachSupportedId().length > 0 ? describe : describe.skip;
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20240523100528_init.js', async () => {
const knex = await databases.init(databaseId);
maybeDescribe('migrations', () => {
it.each(databases.eachSupportedId())(
'20240523100528_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240523100528_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240523100528_init.js');
await migrateUpOnce(knex);
await knex('event_bus_events').insert({
topic: 'test',
created_by: 'abc',
data_json: JSON.stringify({ message: 'hello' }),
notified_subscribers: ['tester'],
});
await knex('event_bus_subscriptions').insert({
id: 'tester',
created_by: 'abc',
read_until: '5',
topics: ['test', 'test2'],
});
await knex('event_bus_events').insert({
topic: 'test',
await expect(knex('event_bus_events')).resolves.toEqual([
{
id: '1',
created_by: 'abc',
topic: 'test',
data_json: JSON.stringify({ message: 'hello' }),
created_at: expect.anything(),
notified_subscribers: ['tester'],
});
await knex('event_bus_subscriptions').insert({
},
]);
await expect(knex('event_bus_subscriptions')).resolves.toEqual([
{
id: 'tester',
created_by: 'abc',
created_at: expect.anything(),
updated_at: expect.anything(),
read_until: '5',
topics: ['test', 'test2'],
});
},
]);
await expect(knex('event_bus_events')).resolves.toEqual([
{
id: '1',
created_by: 'abc',
topic: 'test',
data_json: JSON.stringify({ message: 'hello' }),
created_at: expect.anything(),
notified_subscribers: ['tester'],
},
]);
await expect(knex('event_bus_subscriptions')).resolves.toEqual([
{
id: 'tester',
created_by: 'abc',
created_at: expect.anything(),
updated_at: expect.anything(),
read_until: '5',
topics: ['test', 'test2'],
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('event_bus_events')).rejects.toEqual(expect.anything());
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('event_bus_events')).rejects.toEqual(expect.anything());
await knex.destroy();
},
);
await knex.destroy();
});
});
plugins/events-backend/src/service/EventsPlugin.test.ts
+236 -258
View File
@@ -22,7 +22,6 @@ import {
} from '@backstage/backend-plugin-api';
import {
TestBackend,
TestDatabaseId,
TestDatabases,
mockCredentials,
mockServices,
@@ -105,8 +104,15 @@ describe('eventsPlugin', () => {
test: 'fake-ext',
});
});
});
describe('event bus', () => {
const eventBusDatabases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'],
});
describe.each(eventBusDatabases.eachSupportedId())(
'event bus, %p',
databaseId => {
class ReqHelper {
private readonly backend: TestBackend;
@@ -146,12 +152,8 @@ describe('eventsPlugin', () => {
}
}
const databases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'],
});
async function mockKnexFactory(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
async function mockKnexFactory() {
const knex = await eventBusDatabases.init(databaseId);
return mockServices.database.factory({ knex });
}
@@ -163,278 +165,254 @@ describe('eventsPlugin', () => {
}
});
it.each(databases.eachSupportedId())(
'should be possible to publish events as a service, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should be possible to publish events as a service', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.service.header())
.expect(204); // 204, since there are no subscribers
},
);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.service.header())
.expect(204); // 204, since there are no subscribers
});
it.each(databases.eachSupportedId())(
'should be possible to subscribe as a service and receive an event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should be possible to subscribe as a service and receive an event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper.subscribe('tester', ['test']).expect(201);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber
await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber
await helper.readEvents('tester').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
},
);
await helper.readEvents('tester').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
});
it.each(databases.eachSupportedId())(
'should only send an event for each subscriber once, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should only send an event for each subscriber once', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// A single event
await helper.publish('test', { n: 1 }).expect(201);
// A single event
await helper.publish('test', { n: 1 }).expect(201);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Two clients for subscriber 2, only one gets the event
const res1 = helper.readEvents('tester-2');
const res2 = helper.readEvents('tester-2');
// Two clients for subscriber 2, only one gets the event
const res1 = helper.readEvents('tester-2');
const res2 = helper.readEvents('tester-2');
const res = await Promise.race([res1, res2]);
expect(res.status).toBe(200);
expect(res.body).toEqual({
events: [{ topic: 'test', payload: { n: 1 } }],
});
const res = await Promise.race([res1, res2]);
expect(res.status).toBe(200);
expect(res.body).toEqual({
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Post another event, which triggers the other client to return
await helper.publish('test', { n: 2 }).expect(201);
// Post another event, which triggers the other client to return
await helper.publish('test', { n: 2 }).expect(201);
const otherRes = await Promise.all([res1, res2]).then(rs =>
rs.find(r => r !== res),
const otherRes = await Promise.all([res1, res2]).then(rs =>
rs.find(r => r !== res),
);
expect(otherRes?.status).toBe(202);
// Reading subscriber 2 should now return the second event only
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { n: 2 } }],
});
});
it('should not notify subscribers that have already consumed the event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// A single event for each subscriber, that should not be sent to the other one
await helper
.publish(
'test',
{ for: 'tester-2' },
{
notifiedSubscribers: ['tester-1'],
},
)
.expect(201);
await helper
.publish(
'test',
{ for: 'tester-1' },
{
notifiedSubscribers: ['tester-2'],
},
)
.expect(201);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-1' } }],
});
// Single client for subscriber 2 gets the event
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-2' } }],
});
});
it('should return multiple events in order', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester', ['test']).expect(201);
// A sequence of events published one at a time
for (let n = 0; n < 15; ++n) {
await helper.publish('test', { n }).expect(201);
}
// Batch size it 10
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 0 } },
{ topic: 'test', payload: { n: 1 } },
{ topic: 'test', payload: { n: 2 } },
{ topic: 'test', payload: { n: 3 } },
{ topic: 'test', payload: { n: 4 } },
{ topic: 'test', payload: { n: 5 } },
{ topic: 'test', payload: { n: 6 } },
{ topic: 'test', payload: { n: 7 } },
{ topic: 'test', payload: { n: 8 } },
{ topic: 'test', payload: { n: 9 } },
],
});
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 10 } },
{ topic: 'test', payload: { n: 11 } },
{ topic: 'test', payload: { n: 12 } },
{ topic: 'test', payload: { n: 13 } },
{ topic: 'test', payload: { n: 14 } },
],
});
});
it('should skip publishing if all subscribers have already consumed the event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.publish(
'test',
{ for: 'tester-2' },
{ notifiedSubscribers: ['tester'] },
)
.expect(204);
});
it('should time out when no events are available', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
jest.useFakeTimers({
advanceTimers: true,
});
try {
// Can't use supertest for this one because it can't handle the partially blocking response
const res = await fetch(
`http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`,
{
headers: {
authorization: mockCredentials.service.header(),
},
},
);
expect(otherRes?.status).toBe(202);
// Reading subscriber 2 should now return the second event only
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { n: 2 } }],
});
},
);
expect(res.status).toBe(202);
it.each(databases.eachSupportedId())(
'should not notify subscribers that have already consumed the event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
const reader = res.body!.getReader();
const isClosed = () =>
Promise.race([
reader
.read()
.then(() => reader.closed)
.then(() => true),
new Promise<boolean>(r => setTimeout(r, 100, false)),
]);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(true);
} finally {
jest.useRealTimers();
}
});
// A single event for each subscriber, that should not be sent to the other one
await helper
.publish(
'test',
{ for: 'tester-2' },
{
notifiedSubscribers: ['tester-1'],
},
)
.expect(201);
await helper
.publish(
'test',
{ for: 'tester-1' },
{
notifiedSubscribers: ['tester-2'],
},
)
.expect(201);
it('should refuse listen without a subscription', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-1' } }],
});
// Single client for subscriber 2 gets the event
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-2' } }],
});
},
);
it.each(databases.eachSupportedId())(
'should return multiple events in order, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester', ['test']).expect(201);
// A sequence of events published one at a time
for (let n = 0; n < 15; ++n) {
await helper.publish('test', { n }).expect(201);
}
// Batch size it 10
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 0 } },
{ topic: 'test', payload: { n: 1 } },
{ topic: 'test', payload: { n: 2 } },
{ topic: 'test', payload: { n: 3 } },
{ topic: 'test', payload: { n: 4 } },
{ topic: 'test', payload: { n: 5 } },
{ topic: 'test', payload: { n: 6 } },
{ topic: 'test', payload: { n: 7 } },
{ topic: 'test', payload: { n: 8 } },
{ topic: 'test', payload: { n: 9 } },
],
});
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 10 } },
{ topic: 'test', payload: { n: 11 } },
{ topic: 'test', payload: { n: 12 } },
{ topic: 'test', payload: { n: 13 } },
{ topic: 'test', payload: { n: 14 } },
],
});
},
);
it.each(databases.eachSupportedId())(
'should skip publishing if all subscribers have already consumed the event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.publish(
'test',
{ for: 'tester-2' },
{ notifiedSubscribers: ['tester'] },
)
.expect(204);
},
);
it.each(databases.eachSupportedId())(
'should time out when no events are available, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
jest.useFakeTimers({
advanceTimers: true,
});
try {
// Can't use supertest for this one because it can't handle the partially blocking response
const res = await fetch(
`http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`,
{
headers: {
authorization: mockCredentials.service.header(),
},
},
);
expect(res.status).toBe(202);
const reader = res.body!.getReader();
const isClosed = () =>
Promise.race([
reader
.read()
.then(() => reader.closed)
.then(() => true),
new Promise<boolean>(r => setTimeout(r, 100, false)),
]);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(true);
} finally {
jest.useRealTimers();
}
},
);
it.each(databases.eachSupportedId())(
'should refuse listen without a subscription, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.readEvents('nonexistent').expect(404);
},
);
});
});
await helper.readEvents('nonexistent').expect(404);
});
},
);
plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts
+15 -30
View File
@@ -29,13 +29,10 @@ const databases = TestDatabases.create({
ids: ['POSTGRES_14', 'POSTGRES_18'],
});
const maybeDescribe =
databases.eachSupportedId().length > 0 ? describe : describe.skip;
maybeDescribe('DatabaseEventBusStore', () => {
it.each(databases.eachSupportedId())(
'should clean up old events, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'DatabaseEventBusStore, %p',
databaseId => {
it('should clean up old events', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({ logger, db });
@@ -79,12 +76,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events3 } = await store.readSubscription('tester-3');
expect(events3.length).toBe(5);
},
);
});
it.each(databases.eachSupportedId())(
'should always clean up events outside the max age window, %p',
async databaseId => {
it('should always clean up events outside the max age window', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -132,12 +126,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events3 } = await store.readSubscription('tester-3');
expect(events3.length).toBe(0);
},
);
});
it.each(databases.eachSupportedId())(
'should not clean up events within the min age window, %p',
async databaseId => {
it('should not clean up events within the min age window', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -162,12 +153,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events1 } = await store.readSubscription('tester-1');
expect(events1.length).toBe(10);
},
);
});
it.each(databases.eachSupportedId())(
'should clean up a large number of events, %p',
async databaseId => {
it('should clean up a large number of events', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -197,12 +185,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
await expect(db('event_bus_events').count()).resolves.toEqual([
{ count: '5' },
]);
},
);
});
it.each(databases.eachSupportedId())(
'should perform well when looking up events by topic, %p',
async databaseId => {
it('should perform well when looking up events by topic', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -245,6 +230,6 @@ maybeDescribe('DatabaseEventBusStore', () => {
]);
expect(duration).toBeLessThan(20);
},
);
});
});
},
);
plugins/notifications-backend/src/tests/migrations.test.ts
+42 -45
View File
@@ -41,59 +41,56 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20221109192547_search_add_original_value_column.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20221109192547_search_add_original_value_column.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250317_addTopic.js');
await migrateUntilBefore(knex, '20250317_addTopic.js');
await knex
.insert({
await knex
.insert({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
})
.into('user_settings');
await migrateUpOnce(knex);
let rows = await knex('user_settings');
let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
})
.into('user_settings');
settings_key_hash:
'73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15',
topic: null,
}),
]),
);
await migrateUpOnce(knex);
await migrateDownOnce(knex);
let rows = await knex('user_settings');
let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
rows = await knex('user_settings');
normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
settings_key_hash:
'73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15',
topic: null,
}),
]),
);
await migrateDownOnce(knex);
rows = await knex('user_settings');
normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
}),
]),
);
},
);
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
}),
]),
);
});
});
plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts
+598 -649
View File
File diff suppressed because it is too large Load Diff
plugins/search-backend-module-pg/src/database/util.test.ts
+33 -41
View File
@@ -18,45 +18,37 @@ import { queryPostgresMajorVersion } from './util';
jest.setTimeout(60_000);
describe('util', () => {
describe('unsupported', () => {
const databases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8'],
});
it.each(databases.eachSupportedId())(
'should fail on get postgres major version, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await expect(
async () => await queryPostgresMajorVersion(knex),
).rejects.toThrow();
},
);
});
describe('supported', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14'],
});
if (databases.eachSupportedId().length < 1) {
// Only execute tests if at least on database engine is available, e.g. if
// not in CI=1. it.each doesn't support an empty array.
return;
}
it.each(databases.eachSupportedId())(
'should get postgres major version, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const expectedVersion = +databaseId.slice(9);
await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
expectedVersion,
);
},
);
});
const unsupportedDatabases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8'],
});
describe.each(unsupportedDatabases.eachSupportedId())(
'util unsupported, %p',
databaseId => {
it('should fail on get postgres major version', async () => {
const knex = await unsupportedDatabases.init(databaseId);
await expect(
async () => await queryPostgresMajorVersion(knex),
).rejects.toThrow();
});
},
);
const supportedDatabases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14'],
});
describe.each(supportedDatabases.eachSupportedId())(
'util supported, %p',
databaseId => {
it('should get postgres major version', async () => {
const knex = await supportedDatabases.init(databaseId);
const expectedVersion = +databaseId.slice(9);
await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
expectedVersion,
);
});
},
);