paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

Merge pull request #34267 from backstage/freben/describe-each-databases

Browse Source 
tests: use describe.each for database test iteration
This commit is contained in:
Fredrik Adelöw
2026-05-19 10:24:00 +02:00
committed by GitHub
parent 98f496a1ad 8165184fba
commit a29edc58fb
40 changed files with 4406 additions and 4992 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts
+53 -56
View File
@@ -24,13 +24,13 @@ import { ConfigReader } from '@backstage/config';
jest.setTimeout(60_000);
describe('createPluginKeySource', () => {
const databases = TestDatabases.create();
const mockDir = createMockDirectory();
const databases = TestDatabases.create();
const mockDir = createMockDirectory();
it.each(databases.eachSupportedId())(
'works for implicit database (no config), %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'createPluginKeySource, %p',
databaseId => {
it('works for implicit database (no config)', async () => {
const knex = await databases.init(databaseId);
const getClient = jest.fn(async () => knex);
@@ -61,12 +61,9 @@ describe('createPluginKeySource', () => {
}),
],
});
},
);
});
it.each(databases.eachSupportedId())(
'works for explicit database, %p',
async databaseId => {
it('works for explicit database', async () => {
const knex = await databases.init(databaseId);
const getClient = jest.fn(async () => knex);
@@ -99,11 +96,12 @@ describe('createPluginKeySource', () => {
}),
],
});
},
);
});
},
);
it('works for static', async () => {
const privateKey = `
it('works for static', async () => {
const privateKey = `
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgR8Ja2ppMEgOm1KeY
Kpje00U1luybndt6yC263vcgeKqhRANCAAS+slUrS9JXgtHB1RcDnmlveuu4H3Zm
@@ -111,60 +109,59 @@ describe('createPluginKeySource', () => {
-----END PRIVATE KEY-----
`.trim();
const publicKey = `
const publicKey = `
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvrJVK0vSV4LRwdUXA55pb3rruB92
ZoUEY72HTvjIP9xSbLOhWhMREk84T0q91/m3v3sWq5EzHIWw1zRHQisKZw==
-----END PUBLIC KEY-----
`.trim();
mockDir.setContent({
'public.pem': publicKey,
'private.pem': privateKey,
});
const publicKeyPath = mockDir.resolve('public.pem');
const privateKeyPath = mockDir.resolve('private.pem');
mockDir.setContent({
'public.pem': publicKey,
'private.pem': privateKey,
});
const publicKeyPath = mockDir.resolve('public.pem');
const privateKeyPath = mockDir.resolve('private.pem');
const getClient = jest.fn();
const getClient = jest.fn();
const source = await createPluginKeySource({
config: new ConfigReader({
backend: {
auth: {
pluginKeyStore: {
type: 'static',
static: {
keys: [
{
publicKeyFile: publicKeyPath,
privateKeyFile: privateKeyPath,
keyId: '1',
},
],
},
const source = await createPluginKeySource({
config: new ConfigReader({
backend: {
auth: {
pluginKeyStore: {
type: 'static',
static: {
keys: [
{
publicKeyFile: publicKeyPath,
privateKeyFile: privateKeyPath,
keyId: '1',
},
],
},
},
},
}),
database: mockServices.database.mock({ getClient }),
logger: mockServices.logger.mock(),
keyDuration: { seconds: 10 },
});
},
}),
database: mockServices.database.mock({ getClient }),
logger: mockServices.logger.mock(),
keyDuration: { seconds: 10 },
});
expect(getClient).not.toHaveBeenCalled();
expect(getClient).not.toHaveBeenCalled();
const keys = await source.listKeys();
expect(keys.keys.length).toEqual(1);
expect(keys.keys[0].key).toMatchObject({
kid: '1',
alg: 'ES256',
});
const keys = await source.listKeys();
expect(keys.keys.length).toEqual(1);
expect(keys.keys[0].key).toMatchObject({
kid: '1',
alg: 'ES256',
});
const pk = await source.getPrivateSigningKey();
expect(pk).toMatchObject({
kid: '1',
alg: 'ES256',
d: expect.any(String),
});
const pk = await source.getPrivateSigningKey();
expect(pk).toMatchObject({
kid: '1',
alg: 'ES256',
d: expect.any(String),
});
});
packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts
Vendored
+39 -43
View File
@@ -54,14 +54,16 @@ jest.mock('@keyv/memcache', () => {
};
});
describe('CacheManager integration', () => {
const caches = TestCaches.create();
jest.setTimeout(60_000);
afterEach(jest.clearAllMocks);
const caches = TestCaches.create();
it.each(caches.eachSupportedId())(
'only creates one underlying connection per plugin, %p',
async cacheId => {
describe.each(caches.eachSupportedId())(
'CacheManager integration, %p',
cacheId => {
afterEach(jest.clearAllMocks);
it('only creates one underlying connection per plugin', async () => {
const { store, connection } = await caches.init(cacheId);
const manager = CacheManager.fromConfig(
@@ -85,12 +87,9 @@ describe('CacheManager integration', () => {
// eslint-disable-next-line jest/no-conditional-expect
expect(KeyvValkey).toHaveBeenCalledTimes(3);
}
},
);
});
it.each(caches.eachSupportedId())(
'interacts correctly with store, %p',
async cacheId => {
it('interacts correctly with store', async () => {
const { store, connection } = await caches.init(cacheId);
const manager = CacheManager.fromConfig(
@@ -114,12 +113,9 @@ describe('CacheManager integration', () => {
await expect(plugin1.get('a')).resolves.toBe('plugin1');
await expect(plugin2a.get('a')).resolves.toBe('plugin2b');
await expect(plugin2b.get('a')).resolves.toBe('plugin2b');
},
);
});
it.each(caches.eachSupportedId())(
'supports both milliseconds and human durations throughout, %p',
async cacheId => {
it('supports both milliseconds and human durations throughout', async () => {
const { store, connection } = await caches.init(cacheId);
for (const defaultTtl of [200, { milliseconds: 200 }]) {
@@ -175,39 +171,39 @@ describe('CacheManager integration', () => {
await expect(defaultClient.get('e')).resolves.toBeUndefined();
await expect(defaultClient.get('f')).resolves.toBeUndefined();
}
},
);
});
},
);
it('rejects invalid defaultTtl', () => {
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
},
it('rejects invalid defaultTtl', () => {
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
},
},
}),
),
).not.toThrow();
},
}),
),
).not.toThrow();
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
defaultTtl: 'hello',
},
expect(() =>
CacheManager.fromConfig(
mockServices.rootConfig({
data: {
backend: {
cache: {
store: 'memory',
defaultTtl: 'hello',
},
},
}),
),
).toThrow(/Invalid duration 'hello' in config/);
});
},
}),
),
).toThrow(/Invalid duration 'hello' in config/);
});
describe('CacheManager store options', () => {
packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts
+51 -54
View File
@@ -41,68 +41,65 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20250411000000_last_run.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20250411000000_last_run.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250411000000_last_run.js');
await migrateUntilBefore(knex, '20250411000000_last_run.js');
await knex
.insert({
id: 'i',
settings_json: '{}',
})
.into('backstage_backend_tasks__tasks');
await knex
.insert({
id: 'i',
settings_json: '{}',
})
.into('backstage_backend_tasks__tasks');
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateUpOnce(knex);
await migrateUpOnce(knex);
await knex
.table('backstage_backend_tasks__tasks')
.update({ last_run_error_json: 'error' })
.where({ id: 'i' });
await knex
.table('backstage_backend_tasks__tasks')
.update({ last_run_error_json: 'error' })
.where({ id: 'i' });
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
last_run_ended_at: null,
last_run_error_json: 'error',
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
last_run_ended_at: null,
last_run_error_json: 'error',
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'i',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await knex.destroy();
},
);
await knex.destroy();
});
});
packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts
+49 -54
View File
@@ -24,9 +24,10 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('TaskScheduler', () => {
const databases = TestDatabases.create();
describe.each(databases.eachSupportedId())('TaskScheduler, %p', databaseId => {
const logger = mockServices.logger.mock();
const databases = TestDatabases.create();
const rootLifecycle = mockServices.rootLifecycle.mock();
const httpRouter = mockServices.httpRouter.mock();
const pluginMetadata = {
@@ -35,63 +36,57 @@ describe('TaskScheduler', () => {
const testScopedSignal = createTestScopedSignal();
const metrics = metricsServiceMock.mock();
it.each(databases.eachSupportedId())(
'can return a working v1 plugin impl, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
it('can return a working v1 plugin impl', async () => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromMillis(5000),
signal: testScopedSignal(),
fn,
});
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromMillis(5000),
signal: testScopedSignal(),
fn,
});
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
},
);
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
});
it.each(databases.eachSupportedId())(
'can return a working v2 plugin impl, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
it('can return a working v2 plugin impl', async () => {
const knex = await databases.init(databaseId);
const database = mockServices.database({ knex });
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
const manager = DefaultSchedulerService.create({
database,
logger,
metrics,
rootLifecycle,
httpRouter,
pluginMetadata,
});
const fn = jest.fn();
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
signal: testScopedSignal(),
fn,
});
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
signal: testScopedSignal(),
fn,
});
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
},
);
await waitForExpect(() => {
expect(fn).toHaveBeenCalled();
});
});
});
packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts
+220 -246
View File
@@ -31,48 +31,50 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('PluginTaskManagerImpl', () => {
const addShutdownHook = jest.fn();
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
beforeAll(async () => {
// Make sure all databases are running before mocking timers, in case of testcontainers
await Promise.all(
databases.eachSupportedId().map(([id]) => databases.init(id)),
);
describe.each(databases.eachSupportedId())(
'PluginTaskManagerImpl, %p',
databaseId => {
const addShutdownHook = jest.fn();
jest.useFakeTimers();
}, 60_000);
beforeAll(async () => {
// Make sure the database is running before mocking timers, in case of testcontainers
await databases.init(databaseId);
jest.useFakeTimers();
}, 60_000);
beforeEach(() => {
jest.clearAllMocks();
});
afterAll(() => {
jest.useRealTimers();
});
async function init(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await migrateBackendTasks(knex);
const manager = new PluginTaskSchedulerImpl(
'myplugin',
async () => knex,
mockServices.logger.mock(),
metricsServiceMock.mock(),
{
addShutdownHook,
addBeforeShutdownHook: jest.fn(),
addStartupHook: jest.fn(),
},
);
return { knex, manager };
}
beforeEach(() => {
jest.clearAllMocks();
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can run the v1 happy path, %p',
async databaseId => {
async function init(id: TestDatabaseId) {
const knex = await databases.init(id);
await migrateBackendTasks(knex);
const manager = new PluginTaskSchedulerImpl(
'myplugin',
async () => knex,
mockServices.logger.mock(),
metricsServiceMock.mock(),
{
addShutdownHook,
addBeforeShutdownHook: jest.fn(),
addStartupHook: jest.fn(),
},
);
return { knex, manager };
}
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with global scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -87,12 +89,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'can run the v2 happy path, %p',
async databaseId => {
it('can run the v2 happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -107,12 +106,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'aborts the task if shutdown hook is invoked, %p',
async databaseId => {
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -134,14 +130,11 @@ describe('PluginTaskManagerImpl', () => {
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
},
);
});
});
});
describe('triggerTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can manually trigger a task, %p',
async databaseId => {
describe('triggerTask with global scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -160,12 +153,9 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
it.each(databases.eachSupportedId())(
'cant trigger a non-existent task, %p',
async databaseId => {
it('cant trigger a non-existent task', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -180,12 +170,9 @@ describe('PluginTaskManagerImpl', () => {
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
},
);
});
it.each(databases.eachSupportedId())(
'cant trigger a running task, %p',
async databaseId => {
it('cant trigger a running task', async () => {
const { manager } = await init(databaseId);
const promise = createDeferred();
@@ -205,140 +192,137 @@ describe('PluginTaskManagerImpl', () => {
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
},
);
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with local scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: { milliseconds: 5000 },
frequency: { milliseconds: 5000 },
fn,
scope: 'local',
});
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('scheduleTask with local scope', () => {
it('can run the v1 happy path', async () => {
const { manager } = await init('SQLITE_3');
it('can run the v2 happy path', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: { milliseconds: 5000 },
frequency: { milliseconds: 5000 },
fn,
scope: 'local',
});
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
it('can run the v2 happy path', async () => {
const { manager } = await init('SQLITE_3');
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task2',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
const fn = jest.fn();
const promise = new Promise<AbortSignal>(resolve =>
fn.mockImplementation(resolve),
);
await manager.scheduleTask({
id: 'task3',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
const shutdownHook = addShutdownHook.mock.calls[0][0];
const abortSignal = await promise;
expect(abortSignal.aborted).toBe(false);
it('aborts the task if shutdown hook is invoked', async () => {
const { manager } = await init('SQLITE_3');
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
}, 60_000);
});
const fn = jest.fn();
const promise = new Promise<AbortSignal>(resolve =>
fn.mockImplementation(resolve),
);
await manager.scheduleTask({
id: 'task3',
timeout: Duration.fromMillis(5000),
frequency: { cron: '* * * * * *' },
fn,
scope: 'local',
});
describe('triggerTask with local scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init('SQLITE_3');
const shutdownHook = addShutdownHook.mock.calls[0][0];
const abortSignal = await promise;
expect(abortSignal.aborted).toBe(false);
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
// Should be aborted after the shutdown hook is invoked
await shutdownHook();
expect(abortSignal.aborted).toBe(true);
}, 60_000);
});
await manager.triggerTask('task1');
jest.advanceTimersByTime(5000);
describe('triggerTask with local scope', () => {
it('can manually trigger a task', async () => {
const { manager } = await init('SQLITE_3');
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
const fn = jest.fn();
const promise = new Promise(resolve => fn.mockImplementation(resolve));
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
it('cant trigger a non-existent task', async () => {
const { manager } = await init('SQLITE_3');
await manager.triggerTask('task1');
jest.advanceTimersByTime(5000);
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
}, 60_000);
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
}, 60_000);
it('cant trigger a non-existent task', async () => {
const { manager } = await init('SQLITE_3');
it('cant trigger a running task', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
const promise = createDeferred();
await expect(() => manager.triggerTask('task2')).rejects.toThrow(
NotFoundError,
);
}, 60_000);
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
it('cant trigger a running task', async () => {
const { manager } = await init('SQLITE_3');
await promise;
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
const promise = createDeferred();
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('createScheduledTaskRunner', () => {
it.each(databases.eachSupportedId())(
'can run the happy path, %p',
async databaseId => {
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
await promise;
await expect(() => manager.triggerTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
// This is just to test the wrapper code; most of the actual tests are in
// TaskWorker.test.ts
describe('createScheduledTaskRunner', () => {
it('can run the happy path', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -356,14 +340,11 @@ describe('PluginTaskManagerImpl', () => {
await promise;
expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal));
},
);
});
});
});
describe('can fetch task ids', () => {
it.each(databases.eachSupportedId())(
'can fetch both global and local task ids, %p',
async databaseId => {
describe('can fetch task ids', () => {
it('can fetch both global and local task ids', async () => {
const { manager } = await init(databaseId);
const fn = jest.fn();
@@ -395,52 +376,51 @@ describe('PluginTaskManagerImpl', () => {
settings: expect.objectContaining({ cadence: 'PT5S' }),
},
]);
},
);
});
describe('cancelTask with local scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init('SQLITE_3');
const promise = createDeferred();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
});
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
}, 60_000);
describe('cancelTask with local scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init('SQLITE_3');
it('cannot cancel a task that is not running', async () => {
const { manager } = await init('SQLITE_3');
const promise = createDeferred();
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
fn: async () => {
promise.resolve();
await new Promise(r => setTimeout(r, 20000));
},
scope: 'local',
});
await expect(manager.cancelTask('task1')).rejects.toThrow(ConflictError);
}, 60_000);
});
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
}, 60_000);
describe('cancelTask with global scope', () => {
it.each(databases.eachSupportedId())(
'can cancel a running task, %p',
async databaseId => {
it('cannot cancel a task that is not running', async () => {
const { manager } = await init('SQLITE_3');
const fn = jest.fn();
await manager.scheduleTask({
id: 'task1',
timeout: Duration.fromMillis(5000),
frequency: Duration.fromObject({ years: 1 }),
initialDelay: Duration.fromObject({ years: 1 }),
fn,
scope: 'local',
});
await expect(manager.cancelTask('task1')).rejects.toThrow(
ConflictError,
);
}, 60_000);
});
describe('cancelTask with global scope', () => {
it('can cancel a running task', async () => {
const { manager } = await init(databaseId);
const promise = createDeferred();
@@ -458,23 +438,17 @@ describe('PluginTaskManagerImpl', () => {
await promise;
await expect(manager.cancelTask('task1')).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'cannot cancel a non-existent task, %p',
async databaseId => {
it('cannot cancel a non-existent task', async () => {
const { manager } = await init(databaseId);
await expect(manager.cancelTask('nonexistent')).rejects.toThrow(
NotFoundError,
);
},
);
});
it.each(databases.eachSupportedId())(
'cannot cancel a task that is not running, %p',
async databaseId => {
it('cannot cancel a task that is not running', async () => {
const { manager } = await init(databaseId);
await manager.scheduleTask({
@@ -489,16 +463,16 @@ describe('PluginTaskManagerImpl', () => {
await expect(manager.cancelTask('task1')).rejects.toThrow(
ConflictError,
);
},
);
});
describe('parseDuration', () => {
it('should parse durations', () => {
expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S');
expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S');
expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *');
expect(parseDuration({ trigger: 'manual' })).toEqual('manual');
});
});
});
});
describe('parseDuration', () => {
it('should parse durations', () => {
expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S');
expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S');
expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *');
expect(parseDuration({ trigger: 'manual' })).toEqual('manual');
});
});
},
);
packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts
+23 -22
View File
@@ -34,28 +34,29 @@ const getTask = async (knex: Knex): Promise<DbTasksRow> => {
return (await knex<DbTasksRow>(DB_TASKS_TABLE))[0];
};
describe('PluginTaskSchedulerJanitor', () => {
const logger = mockServices.logger.mock();
const databases = TestDatabases.create({
ids: [
/* 'MYSQL_8' not supported yet */
'POSTGRES_18',
'POSTGRES_14',
'SQLITE_3',
'MYSQL_8',
],
});
const testScopedSignal = createTestScopedSignal();
jest.setTimeout(60_000);
jest.setTimeout(60_000);
const databases = TestDatabases.create({
ids: [
/* 'MYSQL_8' not supported yet */
'POSTGRES_18',
'POSTGRES_14',
'SQLITE_3',
'MYSQL_8',
],
});
beforeEach(() => {
jest.resetAllMocks();
});
describe.each(databases.eachSupportedId())(
'PluginTaskSchedulerJanitor, %p',
databaseId => {
const logger = mockServices.logger.mock();
const testScopedSignal = createTestScopedSignal();
it.each(databases.eachSupportedId())(
'Should update date if current_run_expires_at expires, %p',
async databaseId => {
beforeEach(() => {
jest.resetAllMocks();
});
it('Should update date if current_run_expires_at expires', async () => {
const knex = await databases.init(databaseId);
await migrateBackendTasks(knex);
@@ -92,6 +93,6 @@ describe('PluginTaskSchedulerJanitor', () => {
}),
);
});
},
);
});
});
},
);
packages/backend-defaults/src/migrations.test.ts
+57 -63
View File
@@ -41,83 +41,77 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20210928160613_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20210928160613_init.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20210928160613_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20210928160613_init.js');
await migrateUpOnce(knex);
await knex('backstage_backend_tasks__tasks').insert({
await knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.fn.now(),
});
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: knex.fn.now(),
});
next_run_start_at: expect.anything(),
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: expect.anything(),
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual(
expect.anything(),
);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual(
expect.anything(),
);
await knex.destroy();
});
await knex.destroy();
},
);
it('20240712211735_nullable_next_run.js', async () => {
const knex = await databases.init(databaseId);
it.each(databases.eachSupportedId())(
'20240712211735_nullable_next_run.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js');
await migrateUpOnce(knex);
await knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
});
await knex('backstage_backend_tasks__tasks').insert({
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await expect(
knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
});
}),
).rejects.toEqual(expect.anything());
await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([
{
id: 'test',
settings_json: '{}',
next_run_start_at: null,
current_run_ticket: null,
current_run_started_at: null,
current_run_expires_at: null,
},
]);
await migrateDownOnce(knex);
await expect(
knex('backstage_backend_tasks__tasks').insert({
id: 'test',
settings_json: '{}',
next_run_start_at: knex.raw('null'),
}),
).rejects.toEqual(expect.anything());
await knex.destroy();
},
);
await knex.destroy();
});
});
packages/backend-test-utils/src/cache/TestCaches.test.ts
Vendored
+3 -3
View File
@@ -21,10 +21,10 @@ const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
jest.setTimeout(60_000);
describe('TestCaches', () => {
const caches = TestCaches.create();
const caches = TestCaches.create();
it.each(caches.eachSupportedId())('fires up a cache, %p', async cacheId => {
describe.each(caches.eachSupportedId())('TestCaches, %p', cacheId => {
it('fires up a cache', async () => {
const { keyv } = await caches.init(cacheId);
await keyv.set('test', 'value');
await expect(keyv.get('test')).resolves.toBe('value');
packages/backend-test-utils/src/database/TestDatabases.test.ts
+8 -18
View File
@@ -18,24 +18,14 @@ import { TestDatabases } from './TestDatabases';
jest.setTimeout(60_000);
describe('TestDatabases', () => {
describe('each create', () => {
const dbs = TestDatabases.create();
const dbs = TestDatabases.create();
it.each(dbs.eachSupportedId())(
'creates distinct %p databases',
async databaseId => {
if (!dbs.supports(databaseId)) {
return;
}
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([
{ a: 1 },
]);
},
);
describe.each(dbs.eachSupportedId())('TestDatabases, %p', databaseId => {
it('creates distinct databases', async () => {
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([{ a: 1 }]);
});
});
plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
+14 -23
View File
@@ -19,12 +19,12 @@ import { StaticAssetsStore } from './StaticAssetsStore';
jest.setTimeout(60_000);
describe('StaticAssetsStore', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'should store and retrieve assets, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'StaticAssetsStore, %p',
databaseId => {
it('should store and retrieve assets', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -61,12 +61,9 @@ describe('StaticAssetsStore', () => {
await expect(
store.getAsset('does-not-exist.txt'),
).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'should update assets timestamps, but not contents, %p',
async databaseId => {
it('should update assets timestamps, but not contents', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -112,12 +109,9 @@ describe('StaticAssetsStore', () => {
const sameBar = await store.getAsset('bar');
expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt);
},
);
});
it.each(databases.eachSupportedId())(
'should trim old assets, %p',
async databaseId => {
it('should trim old assets', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -157,12 +151,9 @@ describe('StaticAssetsStore', () => {
await expect(store.getAsset('new')).resolves.toBeDefined();
await expect(store.getAsset('old')).resolves.toBeUndefined();
},
);
});
it.each(databases.eachSupportedId())(
'should isolate assets in namespace, %p',
async databaseId => {
it('should isolate assets in namespace', async () => {
const knex = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
@@ -197,6 +188,6 @@ describe('StaticAssetsStore', () => {
await otherStore.trimAssets({ maxAgeSeconds: 0 });
await expect(otherStore.getAsset('bar')).resolves.not.toBeDefined();
},
);
});
});
},
);
plugins/app-backend/src/migrations.test.ts
+69 -75
View File
@@ -41,101 +41,95 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20211229105307_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20211229105307_init.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20211229105307_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20211229105307_init.js');
await migrateUpOnce(knex);
await knex('static_assets_cache').insert({
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('static_assets_cache')).rejects.toEqual(
expect.anything(),
);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('static_assets_cache')).rejects.toEqual(
expect.anything(),
);
await knex.destroy();
});
await knex.destroy();
},
);
it('20240113144027_assets-namespace.js', async () => {
const knex = await databases.init(databaseId);
it.each(databases.eachSupportedId())(
'20240113144027_assets-namespace.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240113144027_assets-namespace.js');
await migrateUntilBefore(knex, '20240113144027_assets-namespace.js');
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
await knex('static_assets_cache').insert({
await migrateUpOnce(knex);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: knex.fn.now(),
});
namespace: 'default',
last_modified_at: expect.anything(),
},
]);
await migrateUpOnce(knex);
await knex('static_assets_cache').insert({
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: knex.fn.now(),
});
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
]);
await knex('static_assets_cache').insert({
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
{
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: knex.fn.now(),
});
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
namespace: 'default',
last_modified_at: expect.anything(),
},
{
path: 'main.js',
content: Buffer.from('other-script'),
namespace: 'other',
last_modified_at: expect.anything(),
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await expect(knex('static_assets_cache')).resolves.toEqual([
{
path: 'main.js',
content: Buffer.from('some-script'),
last_modified_at: expect.anything(),
},
]);
await knex.destroy();
},
);
await knex.destroy();
});
});
plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts
+10 -13
View File
@@ -27,12 +27,12 @@ const keyBase = {
jest.setTimeout(60_000);
describe('DatabaseKeyStore', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'should store a key, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'DatabaseKeyStore, %p',
databaseId => {
it('should store a key', async () => {
const knex = await databases.init(databaseId);
await AuthDatabase.runMigrations(knex);
@@ -53,12 +53,9 @@ describe('DatabaseKeyStore', () => {
DateTime.fromJSDate(items[0].createdAt).diffNow('seconds').seconds,
),
).toBeLessThan(10);
},
);
});
it.each(databases.eachSupportedId())(
'should remove stored keys, %p',
async databaseId => {
it('should remove stored keys', async () => {
const knex = await databases.init(databaseId);
await AuthDatabase.runMigrations(knex);
@@ -124,6 +121,6 @@ describe('DatabaseKeyStore', () => {
await expect(store.listKeys()).resolves.toEqual({
items: [],
});
},
);
});
});
},
);
plugins/auth-backend/src/identity/KeyStores.test.ts
+66 -75
View File
@@ -24,89 +24,80 @@ import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
jest.setTimeout(60_000);
describe('KeyStores', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
const defaultConfigOptions = {
auth: {
keyStore: {
provider: 'memory',
},
const defaultConfigOptions = {
auth: {
keyStore: {
provider: 'memory',
},
};
const defaultConfig = new ConfigReader(defaultConfigOptions);
},
};
const defaultConfig = new ConfigReader(defaultConfigOptions);
it.each(databases.eachSupportedId())(
'reads auth section from config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig');
const keyStore = await KeyStores.fromConfig(defaultConfig, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
describe.each(databases.eachSupportedId())('KeyStores, %p', databaseId => {
it('reads auth section from config', async () => {
const knex = await databases.init(databaseId);
const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig');
const keyStore = await KeyStores.fromConfig(defaultConfig, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(MemoryKeyStore);
expect(configSpy).toHaveBeenCalledWith('auth.keyStore');
expect(
defaultConfig
.getOptionalConfig('auth.keyStore')
?.getOptionalString('provider'),
).toBe(defaultConfigOptions.auth.keyStore.provider);
},
);
expect(keyStore).toBeInstanceOf(MemoryKeyStore);
expect(configSpy).toHaveBeenCalledWith('auth.keyStore');
expect(
defaultConfig
.getOptionalConfig('auth.keyStore')
?.getOptionalString('provider'),
).toBe(defaultConfigOptions.auth.keyStore.provider);
});
it.each(databases.eachSupportedId())(
'can handle without auth config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const keyStore = await KeyStores.fromConfig(new ConfigReader({}), {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(DatabaseKeyStore);
},
);
it('can handle without auth config', async () => {
const knex = await databases.init(databaseId);
const keyStore = await KeyStores.fromConfig(new ConfigReader({}), {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(DatabaseKeyStore);
});
it.each(databases.eachSupportedId())(
'can handle additional provider config, %p',
async databaseId => {
const knex = await databases.init(databaseId);
jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue();
const createSpy = jest.spyOn(FirestoreKeyStore, 'create');
it('can handle additional provider config', async () => {
const knex = await databases.init(databaseId);
jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue();
const createSpy = jest.spyOn(FirestoreKeyStore, 'create');
const configOptions = {
auth: {
keyStore: {
provider: 'firestore',
firestore: {
projectId: 'my-project',
keyFilename: 'cred.json',
path: 'my-path',
timeout: 100,
host: 'localhost',
port: 8088,
ssl: false,
},
const configOptions = {
auth: {
keyStore: {
provider: 'firestore',
firestore: {
projectId: 'my-project',
keyFilename: 'cred.json',
path: 'my-path',
timeout: 100,
host: 'localhost',
port: 8088,
ssl: false,
},
},
};
const config = new ConfigReader(configOptions);
const keyStore = await KeyStores.fromConfig(config, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
},
};
const config = new ConfigReader(configOptions);
const keyStore = await KeyStores.fromConfig(config, {
logger: mockServices.logger.mock(),
database: AuthDatabase.create(mockServices.database({ knex })),
});
expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
expect(createSpy).toHaveBeenCalledWith(
configOptions.auth.keyStore.firestore,
);
expect(
config
.getOptionalConfig('auth.keyStore')
?.getOptionalConfig('firestore')
?.getOptionalString('projectId'),
).toBe(configOptions.auth.keyStore.firestore.projectId);
},
);
expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
expect(createSpy).toHaveBeenCalledWith(
configOptions.auth.keyStore.firestore,
);
expect(
config
.getOptionalConfig('auth.keyStore')
?.getOptionalConfig('firestore')
?.getOptionalString('projectId'),
).toBe(configOptions.auth.keyStore.firestore.projectId);
});
});
plugins/auth-backend/src/migrations.test.ts
+330 -350
View File
@@ -41,179 +41,154 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20230428155633_sessions.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20230428155633_sessions.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20230428155633_sessions.js');
await migrateUntilBefore(knex, '20230428155633_sessions.js');
await migrateUpOnce(knex);
// Ensure that large cookies are supported
const data = `{"cookie":"${'a'.repeat(100_000)}"}`;
await knex
.insert({ sid: 'abc', expired: knex.fn.now(), sess: data })
.into('sessions');
await knex
.insert({ sid: 'def', expired: knex.fn.now(), sess: data })
.into('sessions');
await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([
{ sid: 'abc', expired: expect.anything(), sess: data },
{ sid: 'def', expired: expect.anything(), sess: data },
]);
await migrateDownOnce(knex);
await knex.destroy();
});
it('20240510120825_user_info.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240510120825_user_info.js');
await migrateUpOnce(knex);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await expect(knex('user_info')).resolves.toEqual([
{
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: expect.anything(),
},
]);
await migrateDownOnce(knex);
await knex.destroy();
});
it('20250707164600_user_created_at.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250707164600_user_created_at.js');
if (knex.client.config.client.includes('sqlite')) {
// Sqlite doesn't support adding a column with non-constant default when table has data
// so we just test that the migration runs without errors
await migrateUpOnce(knex);
// Ensure that large cookies are supported
const data = `{"cookie":"${'a'.repeat(100_000)}"}`;
await knex
.insert({ sid: 'abc', expired: knex.fn.now(), sess: data })
.into('sessions');
await knex
.insert({ sid: 'def', expired: knex.fn.now(), sess: data })
.into('sessions');
return;
}
await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([
{ sid: 'abc', expired: expect.anything(), sess: data },
{ sid: 'def', expired: expect.anything(), sess: data },
]);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await migrateDownOnce(knex);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await knex.destroy();
},
);
const { exp } = await knex('user_info').first();
it.each(databases.eachSupportedId())(
'20240510120825_user_info.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240510120825_user_info.js');
await migrateUpOnce(knex);
const { created_at, updated_at } = await knex('user_info').first();
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
expect(updated_at).toEqual(exp);
expect(created_at).toBeDefined();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info')
.onConflict(['user_entity_ref'])
.merge();
await expect(knex('user_info')).resolves.toEqual([
{
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: expect.anything(),
},
]);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user-2',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info');
await migrateDownOnce(knex);
await expect(
knex('user_info').select('created_at', 'updated_at'),
).resolves.toEqual([
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
]);
await knex.destroy();
},
);
await migrateDownOnce(knex);
it.each(databases.eachSupportedId())(
'20250707164600_user_created_at.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250707164600_user_created_at.js');
await expect(knex('user_info').select('exp')).resolves.toEqual([
{ exp: expect.any(Date) },
{ exp: expect.any(Date) },
]);
if (knex.client.config.client.includes('sqlite')) {
// Sqlite doesn't support adding a column with non-constant default when table has data
// so we just test that the migration runs without errors
await migrateUpOnce(knex);
await knex.destroy();
});
return;
}
it('20250909120000_oidc_client_registration.js', async () => {
const knex = await databases.init(databaseId);
const user_info = JSON.stringify({
claims: {
ent: ['group:default/group1', 'group:default/group2'],
},
});
await migrateUntilBefore(
knex,
'20250909120000_oidc_client_registration.js',
);
await migrateUpOnce(knex);
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
exp: knex.fn.now(),
})
.into('user_info');
const { exp } = await knex('user_info').first();
await migrateUpOnce(knex);
const { created_at, updated_at } = await knex('user_info').first();
expect(updated_at).toEqual(exp);
expect(created_at).toBeDefined();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info')
.onConflict(['user_entity_ref'])
.merge();
await knex
.insert({
user_entity_ref: 'user:default/backstage-user-2',
user_info,
updated_at: knex.fn.now(),
})
.into('user_info');
await expect(
knex('user_info').select('created_at', 'updated_at'),
).resolves.toEqual([
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
{
created_at: expect.any(Date),
updated_at: expect.any(Date),
},
]);
await migrateDownOnce(knex);
await expect(knex('user_info').select('exp')).resolves.toEqual([
{ exp: expect.any(Date) },
{ exp: expect.any(Date) },
]);
await knex.destroy();
},
);
it.each(databases.eachSupportedId())(
'20250909120000_oidc_client_registration.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(
knex,
'20250909120000_oidc_client_registration.js',
);
await migrateUpOnce(knex);
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
})
.into('oidc_clients');
await expect(
knex('oidc_clients').where('client_id', 'test-client-id').first(),
).resolves.toEqual({
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
@@ -222,230 +197,235 @@ describe('migrations', () => {
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
});
})
.into('oidc_clients');
await knex
.insert({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await expect(
knex('oidc_clients').where('client_id', 'test-client-id').first(),
).resolves.toEqual({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
scope: 'openid profile',
metadata: JSON.stringify({ description: 'Test client' }),
});
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
}),
);
await knex
.insert({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex
.insert({
code: 'test-auth-code',
session_id: 'test-session-id',
expires_at: new Date(Date.now() + 600000),
used: false,
})
.into('oidc_authorization_codes');
await expect(
knex('oidc_authorization_codes')
.where('code', 'test-auth-code')
.first(),
).resolves.toEqual(
expect.objectContaining({
code: 'test-auth-code',
session_id: 'test-session-id',
}),
);
await knex('oauth_authorization_sessions')
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.del();
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-session-id',
client_id: 'test-client-id',
user_entity_ref: 'user:default/test-user',
redirect_uri: 'https://example.com/callback',
scope: 'openid',
state: 'test-state',
response_type: 'code',
code_challenge: 'test-challenge',
code_challenge_method: 'S256',
nonce: 'test-nonce',
status: 'pending',
}),
);
await expect(
knex('oidc_authorization_codes').where('session_id', 'test-session-id'),
).resolves.toHaveLength(0);
await knex
.insert({
code: 'test-auth-code',
session_id: 'test-session-id',
expires_at: new Date(Date.now() + 600000),
used: false,
})
.into('oidc_authorization_codes');
await migrateDownOnce(knex);
await expect(
knex('oidc_authorization_codes').where('code', 'test-auth-code').first(),
).resolves.toEqual(
expect.objectContaining({
code: 'test-auth-code',
session_id: 'test-session-id',
}),
);
const tables = [
'oidc_clients',
'oauth_authorization_sessions',
'oidc_authorization_codes',
];
await knex('oauth_authorization_sessions')
.where('id', 'test-session-id')
.del();
for (const table of tables) {
await expect(knex.schema.hasTable(table)).resolves.toBe(false);
}
await expect(
knex('oidc_authorization_codes').where('session_id', 'test-session-id'),
).resolves.toHaveLength(0);
await knex.destroy();
},
);
await migrateDownOnce(knex);
it.each(databases.eachSupportedId())(
'20251118120000_oauth_state_text.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const tables = [
'oidc_clients',
'oauth_authorization_sessions',
'oidc_authorization_codes',
];
await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js');
for (const table of tables) {
await expect(knex.schema.hasTable(table)).resolves.toBe(false);
}
// First create a client for the foreign key constraint
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
await knex.destroy();
});
// Insert a session with state before migration
const existingState = 'existing-short-state';
await knex
.insert({
id: 'test-existing-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: existingState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it('20251118120000_oauth_state_text.js', async () => {
const knex = await databases.init(databaseId);
// Apply the migration that changes state to TEXT
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js');
// Verify existing state persists after migration
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-existing-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-existing-session',
state: existingState,
}),
);
// First create a client for the foreign key constraint
await knex
.insert({
client_id: 'test-client-id',
client_secret: 'test-client-secret',
client_name: 'Test Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
// Test inserting a state parameter longer than 255 characters
// This is based on the real-world example from the issue
const longState = 'a'.repeat(280);
// Insert a session with state before migration
const existingState = 'existing-short-state';
await knex
.insert({
id: 'test-existing-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: existingState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex
.insert({
id: 'test-long-state-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: longState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Apply the migration that changes state to TEXT
await migrateUpOnce(knex);
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-long-state-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-long-state-session',
state: longState,
}),
);
// Verify existing state persists after migration
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-existing-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-existing-session',
state: existingState,
}),
);
await migrateDownOnce(knex);
// Test inserting a state parameter longer than 255 characters
// This is based on the real-world example from the issue
const longState = 'a'.repeat(280);
await knex.destroy();
},
);
await knex
.insert({
id: 'test-long-state-session',
client_id: 'test-client-id',
redirect_uri: 'https://example.com/callback',
state: longState,
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it.each(databases.eachSupportedId())(
'20251217120000_drop_oidc_clients_fk.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await expect(
knex('oauth_authorization_sessions')
.where('id', 'test-long-state-session')
.first(),
).resolves.toEqual(
expect.objectContaining({
id: 'test-long-state-session',
state: longState,
}),
);
await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js');
await migrateDownOnce(knex);
// Create a client for DCR sessions
await knex
.insert({
client_id: 'dcr-client-id',
client_secret: 'test-client-secret',
client_name: 'DCR Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
await knex.destroy();
});
// Create a DCR session (has matching client in oidc_clients)
await knex
.insert({
id: 'dcr-session',
client_id: 'dcr-client-id',
redirect_uri: 'https://example.com/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
it('20251217120000_drop_oidc_clients_fk.js', async () => {
const knex = await databases.init(databaseId);
// Apply migration - drops FK constraint
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js');
// Now we can insert a CIMD session (URL-based client_id not in oidc_clients)
await knex
.insert({
id: 'cimd-session',
client_id: 'https://example.com/.well-known/oauth-client/cli',
redirect_uri: 'http://localhost:8080/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Create a client for DCR sessions
await knex
.insert({
client_id: 'dcr-client-id',
client_secret: 'test-client-secret',
client_name: 'DCR Client',
response_types: JSON.stringify(['code']),
grant_types: JSON.stringify(['authorization_code']),
redirect_uris: JSON.stringify(['https://example.com/callback']),
})
.into('oidc_clients');
// Verify both sessions exist
await expect(
knex('oauth_authorization_sessions').select('id').orderBy('id'),
).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]);
// Create a DCR session (has matching client in oidc_clients)
await knex
.insert({
id: 'dcr-session',
client_id: 'dcr-client-id',
redirect_uri: 'https://example.com/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
// Rollback - should delete CIMD sessions and re-add FK
await migrateDownOnce(knex);
// Apply migration - drops FK constraint
await migrateUpOnce(knex);
// CIMD session should be deleted, DCR session should remain
await expect(
knex('oauth_authorization_sessions').select('id'),
).resolves.toEqual([{ id: 'dcr-session' }]);
// Now we can insert a CIMD session (URL-based client_id not in oidc_clients)
await knex
.insert({
id: 'cimd-session',
client_id: 'https://example.com/.well-known/oauth-client/cli',
redirect_uri: 'http://localhost:8080/callback',
response_type: 'code',
status: 'pending',
expires_at: new Date(Date.now() + 3600000),
})
.into('oauth_authorization_sessions');
await knex.destroy();
},
);
// Verify both sessions exist
await expect(
knex('oauth_authorization_sessions').select('id').orderBy('id'),
).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]);
// Rollback - should delete CIMD sessions and re-add FK
await migrateDownOnce(knex);
// CIMD session should be deleted, DCR session should remain
await expect(
knex('oauth_authorization_sessions').select('id'),
).resolves.toEqual([{ id: 'dcr-session' }]);
await knex.destroy();
});
});
plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts
+18 -17
View File
@@ -23,18 +23,20 @@ const migrationsDir = `${__dirname}/../../migrations`;
jest.setTimeout(60_000);
describe('IncrementalIngestionDatabaseManager', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
'stores and retrieves marks, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'IncrementalIngestionDatabaseManager, %p',
databaseId => {
it('stores and retrieves marks', async () => {
const knex = await databases.init(databaseId);
await knex.migrate.latest({ directory: migrationsDir });
const manager = new IncrementalIngestionDatabaseManager({ client: knex });
const manager = new IncrementalIngestionDatabaseManager({
client: knex,
});
const { ingestionId } = (await manager.createProviderIngestionRecord(
'myProvider',
))!;
@@ -75,16 +77,15 @@ describe('IncrementalIngestionDatabaseManager', () => {
sequence: 1,
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'computeRemoved correctly sums total count from count query, %p',
async databaseId => {
it('computeRemoved correctly sums total count from count query', async () => {
const knex = await databases.init(databaseId);
await knex.migrate.latest({ directory: migrationsDir });
const manager = new IncrementalIngestionDatabaseManager({ client: knex });
const manager = new IncrementalIngestionDatabaseManager({
client: knex,
});
const { ingestionId } = (await manager.createProviderIngestionRecord(
'testProvider',
))!;
@@ -119,6 +120,6 @@ describe('IncrementalIngestionDatabaseManager', () => {
// On PostgreSQL, count queries return strings, so total should be 3 not NaN or string concatenation
expect(result.total).toBe(3);
expect(typeof result.total).toBe('number');
},
);
});
});
},
);
plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts
+20 -19
View File
@@ -23,24 +23,25 @@ import { WrapperProviders } from './WrapperProviders';
jest.setTimeout(60_000);
describe('WrapperProviders', () => {
const applyDatabaseMigrations = jest.fn();
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'],
});
const config = new ConfigReader({});
const logger = mockServices.logger.mock();
const scheduler = {
scheduleTask: jest.fn(),
};
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'],
});
beforeEach(() => {
jest.clearAllMocks();
});
describe.each(databases.eachSupportedId())(
'WrapperProviders, %p',
databaseId => {
const applyDatabaseMigrations = jest.fn();
const config = new ConfigReader({});
const logger = mockServices.logger.mock();
const scheduler = {
scheduleTask: jest.fn(),
};
it.each(databases.eachSupportedId())(
'should initialize the providers in order, %p',
async databaseId => {
beforeEach(() => {
jest.clearAllMocks();
});
it('should initialize the providers in order', async () => {
const client = await databases.init(databaseId);
const provider1: IncrementalEntityProvider<number, {}> = {
@@ -111,6 +112,6 @@ describe('WrapperProviders', () => {
id: 'provider2',
}),
);
},
);
});
});
},
);
plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts
+38 -44
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { DefaultCatalogDatabase } from './DefaultCatalogDatabase';
import { applyDatabaseMigrations } from './migrations';
import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';
@@ -26,48 +22,46 @@ import { LoggerService } from '@backstage/backend-plugin-api';
jest.setTimeout(60_000);
describe('DefaultCatalogDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
describe.each(databases.eachSupportedId())(
'DefaultCatalogDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
describe('listAncestors', () => {
let nextId = 1;
function makeEntity(ref: string) {
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
knex,
db: new DefaultCatalogDatabase({
database: knex,
logger,
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return ancestors, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('listAncestors', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it('should return ancestors', async () => {
const { knex, db } = await createDatabase();
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
@@ -107,7 +101,7 @@ describe('DefaultCatalogDatabase', () => {
'location:default/root-1',
'location:default/root-2',
]);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts
+110 -152
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
@@ -40,64 +36,62 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('DefaultProcessingDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: createRandomProcessingInterval({
minSeconds: 100,
maxSeconds: 150,
describe.each(databases.eachSupportedId())(
'DefaultProcessingDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: createRandomProcessingInterval({
minSeconds: 100,
maxSeconds: 150,
}),
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
};
}
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
describe('updateProcessedEntity', () => {
let id: string;
let processedEntity: Entity;
beforeEach(() => {
id = uuid();
processedEntity = {
apiVersion: '1',
kind: 'Location',
metadata: {
name: 'fakelocation',
},
spec: {
type: 'url',
target: 'somethingelse',
},
};
});
}
it.each(databases.eachSupportedId())(
'fails when an entity is processed with a different locationKey, %p',
async databaseId => {
const { db } = await createDatabase(databaseId);
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
describe('updateProcessedEntity', () => {
let id: string;
let processedEntity: Entity;
beforeEach(() => {
id = uuid();
processedEntity = {
apiVersion: '1',
kind: 'Location',
metadata: {
name: 'fakelocation',
},
spec: {
type: 'url',
target: 'somethingelse',
},
};
});
it('fails when an entity is processed with a different locationKey', async () => {
const { db } = await createDatabase();
await db.transaction(async tx => {
await expect(() =>
db.updateProcessedEntity(tx, {
@@ -112,12 +106,9 @@ describe('DefaultProcessingDatabase', () => {
`Conflicting write of processing result for ${id} with location key 'undefined'`,
);
});
},
);
});
it.each(databases.eachSupportedId())(
'fails when the locationKey is different, %p',
async databaseId => {
it('fails when the locationKey is different', async () => {
const options = {
id,
processedEntity,
@@ -128,7 +119,7 @@ describe('DefaultProcessingDatabase', () => {
refreshKeys: [],
errors: "['something broke']",
};
const { knex, db } = await createDatabase(databaseId);
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -158,13 +149,10 @@ describe('DefaultProcessingDatabase', () => {
`Conflicting write of processing result for ${id} with location key 'fail'`,
),
);
},
);
});
it.each(databases.eachSupportedId())(
'updates the refresh state entry with the cache, processed entity and errors, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('updates the refresh state entry with the cache, processed entity and errors', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -197,13 +185,10 @@ describe('DefaultProcessingDatabase', () => {
);
expect(entities[0].errors).toEqual("['something broke']");
expect(entities[0].location_key).toEqual('key');
},
);
});
it.each(databases.eachSupportedId())(
'removes old relations and stores the new relationships, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('removes old relations and stores the new relationships', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -282,13 +267,10 @@ describe('DefaultProcessingDatabase', () => {
target_entity_ref: 'component:default/foo',
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'adds deferred entities to the refresh_state table to be picked up later, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('adds deferred entities to the refresh_state table to be picked up later', async () => {
const { knex, db } = await createDatabase();
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
@@ -330,19 +312,15 @@ describe('DefaultProcessingDatabase', () => {
.select();
expect(refreshStateEntries).toHaveLength(1);
},
);
});
it.each(databases.eachSupportedId())(
'updates unprocessed entities with varying location keys, %p',
async databaseId => {
it('updates unprocessed entities with varying location keys', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
@@ -479,19 +457,15 @@ describe('DefaultProcessingDatabase', () => {
expect(mockLogger.error).not.toHaveBeenCalled();
}
});
},
);
});
it.each(databases.eachSupportedId())(
'stores the refresh keys for the entity where key length is 255 chars or less',
async databaseId => {
it('stores the refresh keys for the entity where key length is 255 chars or less', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
await insertRefreshStateRow(knex, {
@@ -536,19 +510,15 @@ describe('DefaultProcessingDatabase', () => {
entity_id: id,
key: 'protocol:foo-bar.com',
});
},
);
});
it.each(databases.eachSupportedId())(
'stores the refresh keys for the entity where key length is greater than 255 chars',
async databaseId => {
it('stores the refresh keys for the entity where key length is greater than 255 chars', async () => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
await insertRefreshStateRow(knex, {
@@ -597,15 +567,12 @@ describe('DefaultProcessingDatabase', () => {
entity_id: id,
key: `url:https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-back#sha256:edfb606500d184900e63891e5279d35bf0069ea251e90d15c0a430de6023d905`,
});
},
);
});
});
});
describe('updateEntityCache', () => {
it.each(databases.eachSupportedId())(
'updates the entityCache, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('updateEntityCache', () => {
it('updates the entityCache', async () => {
const { knex, db } = await createDatabase();
const id = '123';
await insertRefreshStateRow(knex, {
entity_id: id,
@@ -644,15 +611,12 @@ describe('DefaultProcessingDatabase', () => {
).select();
expect(entities2.length).toBe(1);
expect(entities2[0].cache).toEqual('{}');
},
);
});
});
});
describe('getProcessableEntities', () => {
it.each(databases.eachSupportedId())(
'should return entities to process, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('getProcessableEntities', () => {
it('should return entities to process', async () => {
const { knex, db } = await createDatabase();
const entity = JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
@@ -696,13 +660,10 @@ describe('DefaultProcessingDatabase', () => {
}),
).resolves.toEqual({ items: [] });
});
},
);
});
it.each(databases.eachSupportedId())(
'should update the next_refresh interval with a timestamp that includes refresh spread, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should update the next_refresh interval with a timestamp that includes refresh spread', async () => {
const { knex, db } = await createDatabase();
const entity = JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
@@ -731,33 +692,30 @@ describe('DefaultProcessingDatabase', () => {
const nextUpdate = timestampToDateTime(result[0].next_update_at);
const nextUpdateDiff = nextUpdate.diff(now, 'seconds');
expect(nextUpdateDiff.seconds).toBeGreaterThanOrEqual(90);
},
);
});
});
});
describe('listParents', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
describe('listParents', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return parents, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should return parents', async () => {
const { knex, db } = await createDatabase();
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
@@ -803,7 +761,7 @@ describe('DefaultProcessingDatabase', () => {
db.listParents(tx, { entityRefs: ['location:default/root-2'] }),
);
expect(result3.entityRefs).toEqual([]);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts
+88 -133
View File
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { mockServices, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
@@ -30,54 +26,52 @@ import { generateStableHash } from './util';
jest.setTimeout(60_000);
describe('DefaultProviderDatabase', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProviderDatabase({
database: knex,
logger,
}),
};
}
describe.each(databases.eachSupportedId())(
'DefaultProviderDatabase, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProviderDatabase({
database: knex,
logger,
}),
};
}
};
describe('replaceUnprocessedEntities', () => {
it.each(databases.eachSupportedId())(
'replaces all existing state correctly for simple dependency chains, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => {
return db<DbRefreshStateReferencesRow>('refresh_state_references').insert(
ref,
);
};
const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
};
describe('replaceUnprocessedEntities', () => {
it('replaces all existing state correctly for simple dependency chains', async () => {
const { knex, db } = await createDatabase();
/*
config -> location:default/root -> location:default/root-1 -> location:default/root-2
database -> location:default/second -> location:default/root-2
@@ -187,13 +181,10 @@ describe('DefaultProviderDatabase', () => {
t.source_key === 'config',
),
).toBeTruthy();
},
);
});
it.each(databases.eachSupportedId())(
'should work for more complex chains, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should work for more complex chains', async () => {
const { knex, db } = await createDatabase();
/*
config -> location:default/root -> location:default/root-1 -> location:default/root-2
config -> location:default/root -> location:default/root-1a -> location:default/root-2
@@ -323,13 +314,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/root-2',
),
).toBeFalsy();
},
);
});
it.each(databases.eachSupportedId())(
'should add new locations using the delta options, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should add new locations using the delta options', async () => {
const { knex, db } = await createDatabase();
// Existing state and references should stay
await createLocations(knex, ['location:default/existing']);
@@ -393,13 +381,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/existing',
),
).toBeTruthy();
},
);
});
it.each(databases.eachSupportedId())(
'should not remove locations that are referenced elsewhere, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should not remove locations that are referenced elsewhere', async () => {
const { knex, db } = await createDatabase();
/*
config-1 -> location:default/root
config-2 -> location:default/root
@@ -443,13 +428,10 @@ describe('DefaultProviderDatabase', () => {
entity_ref: 'location:default/root',
}),
]);
},
);
});
it.each(databases.eachSupportedId())(
'should remove old locations using the delta options, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should remove old locations using the delta options', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, ['location:default/new-root']);
await insertRefRow(knex, {
@@ -492,13 +474,10 @@ describe('DefaultProviderDatabase', () => {
t.target_entity_ref === 'location:default/new-root',
),
).toBeFalsy();
},
);
});
it.each(databases.eachSupportedId())(
'should update the location key during full replace, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should update the location key during full replace', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, ['location:default/removed']);
await insertRefreshStateRow(knex, {
entity_id: uuid(),
@@ -558,13 +537,10 @@ describe('DefaultProviderDatabase', () => {
target_entity_ref: 'location:default/replaced',
}),
]);
},
);
});
it.each(databases.eachSupportedId())(
'should support replacing modified entities during a full update, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('should support replacing modified entities during a full update', async () => {
const { knex, db } = await createDatabase();
await db.transaction(async tx => {
await db.replaceUnprocessedEntities(tx, {
@@ -689,14 +665,11 @@ describe('DefaultProviderDatabase', () => {
target_entity_ref: 'component:default/a',
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'should successfully fall back from batch to individual mode on conflicts, %p',
async databaseId => {
it('should successfully fall back from batch to individual mode on conflicts', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
await createLocations(knex, ['component:default/a']);
@@ -738,14 +711,11 @@ describe('DefaultProviderDatabase', () => {
}),
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync, %p',
async databaseId => {
it('should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
await createLocations(knex, ['component:default/a']);
@@ -768,14 +738,11 @@ describe('DefaultProviderDatabase', () => {
const state = await knex<DbRefreshStateRow>('refresh_state').select();
expect(state).toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'should properly translate deltas into add/update/remove, %p',
async databaseId => {
it('should properly translate deltas into add/update/remove', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
const entity1Before: Entity = {
apiVersion: '1',
@@ -950,14 +917,11 @@ describe('DefaultProviderDatabase', () => {
location_key: 'new', // managed to update only the location key
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'can handle large deltas without exploding, %p',
async databaseId => {
it('can handle large deltas without exploding', async () => {
const fakeLogger = mockServices.logger.mock();
const { knex, db } = await createDatabase(databaseId, fakeLogger);
const { knex, db } = await createDatabase(fakeLogger);
const count = 10000;
const padded = (n: number) => String(n).padStart(8, '0');
@@ -989,15 +953,12 @@ describe('DefaultProviderDatabase', () => {
unprocessed_entity: JSON.stringify(entities[0].entity),
unprocessed_hash: generateStableHash(entities[0].entity),
});
},
);
});
});
});
describe('listReferenceSourceKeys', () => {
it.each(databases.eachSupportedId())(
'returns the source_keys from "refresh_state_references", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
describe('listReferenceSourceKeys', () => {
it('returns the source_keys from "refresh_state_references"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1018,13 +979,10 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['bar', 'foo']);
},
);
});
it.each(databases.eachSupportedId())(
'returns only unique source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('returns only unique source_keys"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1045,13 +1003,10 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['foo']);
},
);
});
it.each(databases.eachSupportedId())(
'does not return null source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
it('does not return null source_keys"', async () => {
const { knex, db } = await createDatabase();
await createLocations(knex, [
'location:default/root',
@@ -1071,7 +1026,7 @@ describe('DefaultProviderDatabase', () => {
);
expect(res).toEqual(['foo']);
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/database/metrics.test.ts
+84 -93
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from './migrations';
@@ -23,10 +23,10 @@ import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics';
jest.setTimeout(60_000);
describe('metrics', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
describe.each(databases.eachSupportedId())('metrics, %p', databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
@@ -55,108 +55,99 @@ describe('metrics', () => {
}
describe('queryEntitiesCountByKind', () => {
it.each(databases.eachSupportedId())(
'counts entities grouped by the kind in entity_ref, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
it('counts entities grouped by the kind in entity_ref', async () => {
const knex = await createDatabase();
await insertEntity(knex, {
entityRef: 'component:default/svc-a',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'component:default/svc-b',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'api:default/api-a',
finalEntity: '{"kind":"API"}',
});
await insertEntity(knex, {
entityRef: 'system:other/sys-a',
finalEntity: '{"kind":"System"}',
});
// Not yet stitched -- must be excluded from the count
await insertEntity(knex, {
entityRef: 'component:default/pending',
finalEntity: null,
});
await insertEntity(knex, {
entityRef: 'component:default/svc-a',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'component:default/svc-b',
finalEntity: '{"kind":"Component"}',
});
await insertEntity(knex, {
entityRef: 'api:default/api-a',
finalEntity: '{"kind":"API"}',
});
await insertEntity(knex, {
entityRef: 'system:other/sys-a',
finalEntity: '{"kind":"System"}',
});
// Not yet stitched -- must be excluded from the count
await insertEntity(knex, {
entityRef: 'component:default/pending',
finalEntity: null,
});
const result = await queryEntitiesCountByKind(knex);
const result = await queryEntitiesCountByKind(knex);
expect(Object.fromEntries(result)).toEqual({
component: 2,
api: 1,
system: 1,
});
},
);
expect(Object.fromEntries(result)).toEqual({
component: 2,
api: 1,
system: 1,
});
});
});
describe('createEntitiesCountByKind', () => {
it.each(databases.eachSupportedId())(
'serves cached results within the TTL and refreshes after, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
it('serves cached results within the TTL and refreshes after', async () => {
const knex = await createDatabase();
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
const first = await getCount();
expect(Object.fromEntries(first)).toEqual({ component: 1 });
const first = await getCount();
expect(Object.fromEntries(first)).toEqual({ component: 1 });
// A change made within the TTL window must not be visible yet.
await insertEntity(knex, {
entityRef: 'component:default/two',
finalEntity: '{}',
});
const cached = await getCount();
expect(Object.fromEntries(cached)).toEqual({ component: 1 });
// A change made within the TTL window must not be visible yet.
await insertEntity(knex, {
entityRef: 'component:default/two',
finalEntity: '{}',
});
const cached = await getCount();
expect(Object.fromEntries(cached)).toEqual({ component: 1 });
// After the TTL elapses the next call hits the database again.
await new Promise(resolve => setTimeout(resolve, 80));
const refreshed = await getCount();
expect(Object.fromEntries(refreshed)).toEqual({ component: 2 });
},
);
// After the TTL elapses the next call hits the database again.
await new Promise(resolve => setTimeout(resolve, 80));
const refreshed = await getCount();
expect(Object.fromEntries(refreshed)).toEqual({ component: 2 });
});
it.each(databases.eachSupportedId())(
'coalesces overlapping callers into a single underlying query, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
it('coalesces overlapping callers into a single underlying query', async () => {
const knex = await createDatabase();
const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
await insertEntity(knex, {
entityRef: 'component:default/one',
finalEntity: '{}',
});
const finalEntitiesQueries: string[] = [];
knex.on('query', (q: { sql: string }) => {
if (
/from\s+["`]?final_entities["`]?/i.test(q.sql) &&
/^\s*select/i.test(q.sql)
) {
finalEntitiesQueries.push(q.sql);
}
});
// Five concurrent callers should result in one query, not five.
const results = await Promise.all([
getCount(),
getCount(),
getCount(),
getCount(),
getCount(),
]);
expect(finalEntitiesQueries).toHaveLength(1);
for (const r of results) {
expect(Object.fromEntries(r)).toEqual({ component: 1 });
const finalEntitiesQueries: string[] = [];
knex.on('query', (q: { sql: string }) => {
if (
/from\s+["`]?final_entities["`]?/i.test(q.sql) &&
/^\s*select/i.test(q.sql)
) {
finalEntitiesQueries.push(q.sql);
}
},
);
});
// Five concurrent callers should result in one query, not five.
const results = await Promise.all([
getCount(),
getCount(),
getCount(),
getCount(),
getCount(),
]);
expect(finalEntitiesQueries).toHaveLength(1);
for (const r of results) {
expect(Object.fromEntries(r)).toEqual({ component: 1 });
}
});
});
});
plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts
+79 -94
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from '../../migrations';
@@ -27,72 +27,72 @@ import { deleteWithEagerPruningOfChildren } from './deleteWithEagerPruningOfChil
jest.setTimeout(60_000);
describe('deleteWithEagerPruningOfChildren', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
return knex<DbRefreshStateReferencesRow>('refresh_state_references').insert(
refs,
);
}
async function insertRelation(
knex: Knex,
...relations: { from: string; to: string }[]
) {
for (const rel of relations) {
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: await knex<DbRefreshStateRow>('refresh_state')
.select('entity_id')
.then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion
source_entity_ref: rel.from,
target_entity_ref: rel.to,
type: 'fake',
});
describe.each(databases.eachSupportedId())(
'deleteWithEagerPruningOfChildren, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
return knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert(refs);
}
}
async function remainingEntities(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref');
return rows.map(r => r.entity_ref);
}
async function insertRelation(
knex: Knex,
...relations: { from: string; to: string }[]
) {
for (const rel of relations) {
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: await knex<DbRefreshStateRow>('refresh_state')
.select('entity_id')
.then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion
source_entity_ref: rel.from,
target_entity_ref: rel.to,
type: 'fake',
});
}
}
async function entitiesMarkedForStitching(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref')
.where('result_hash', '=', 'force-stitching');
return rows.map(r => r.entity_ref);
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
}
it.each(databases.eachSupportedId())(
'works for the simple path, %p',
async databaseId => {
async function remainingEntities(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref');
return rows.map(r => r.entity_ref);
}
async function entitiesMarkedForStitching(knex: Knex) {
const rows = await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref')
.where('result_hash', '=', 'force-stitching');
return rows.map(r => r.entity_ref);
}
it('works for the simple path', async () => {
/*
P1 - E1 - E2
@@ -106,7 +106,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1, E2, and E3 deleted; E4 and E5 remain; E4 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5');
await insertReference(
knex,
@@ -124,12 +124,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E4', 'E5']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E4']);
},
);
});
it.each(databases.eachSupportedId())(
'works when there are multiple identical references, %p',
async databaseId => {
it('works when there are multiple identical references', async () => {
/*
P1
\
@@ -143,7 +140,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 remains; E2 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2');
await insertReference(
knex,
@@ -159,12 +156,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']);
},
);
});
it.each(databases.eachSupportedId())(
'leaves out things that have roots in other source keys, %p',
async databaseId => {
it('leaves out things that have roots in other source keys', async () => {
/*
P1 - E1
\
@@ -176,7 +170,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 and E3 remain; E2 marked for stitching because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3');
await insertReference(
knex,
@@ -197,12 +191,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']);
},
);
});
it.each(databases.eachSupportedId())(
'leaves out things that have several different roots for the same source key, %p',
async databaseId => {
it('leaves out things that have several different roots for the same source key', async () => {
/*
P1 - E1
\
@@ -214,7 +205,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E1 deleted; E2 and E3 remain
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3');
await insertReference(
knex,
@@ -230,12 +221,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'handles cycles and diamonds gracefully, %p',
async databaseId => {
it('handles cycles and diamonds gracefully', async () => {
/*
P1 - E1 <-> E2
\
@@ -247,7 +235,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: Everything deleted, but in two steps; E4 marked for stitching in the first step because it had a relation to a deleted entity
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5', 'E6');
await insertReference(
knex,
@@ -281,12 +269,9 @@ describe('deleteWithEagerPruningOfChildren', () => {
});
await expect(remainingEntities(knex)).resolves.toEqual([]);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'silently ignores attempts to delete things that are not your own and/or are not roots, %p',
async databaseId => {
it('silently ignores attempts to delete things that are not your own and/or are not roots', async () => {
/*
P1 - E1 - E2
@@ -298,7 +283,7 @@ describe('deleteWithEagerPruningOfChildren', () => {
Result: E3 is deleted; E1, E2 and E4 remain
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(knex, 'E1', 'E2', 'E3', 'E4');
await insertReference(
knex,
@@ -318,6 +303,6 @@ describe('deleteWithEagerPruningOfChildren', () => {
'E4',
]);
await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts
+15 -15
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { randomUUID as uuid } from 'node:crypto';
import { applyDatabaseMigrations } from '../../migrations';
import { DbRefreshKeysRow, DbRefreshStateRow } from '../../tables';
@@ -23,19 +23,19 @@ import { refreshByRefreshKeys } from './refreshByRefreshKeys';
jest.setTimeout(60_000);
describe('refreshByRefreshKeys', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
describe.each(databases.eachSupportedId())(
'refreshByRefreshKeys, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
it.each(databases.eachSupportedId())(
'works for the simple path, %p',
async databaseId => {
const knex = await createDatabase(databaseId);
it('works for the simple path', async () => {
const knex = await createDatabase();
const eid1 = uuid();
await knex<DbRefreshStateRow>('refresh_state').insert({
@@ -89,6 +89,6 @@ describe('refreshByRefreshKeys', () => {
expect(normalizeTimestamp(before2.next_update_at)).toEqual(
normalizeTimestamp(after2.next_update_at),
);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts
+8 -8
View File
@@ -21,12 +21,12 @@ import { getDeferredStitchableEntities } from './getDeferredStitchableEntities';
jest.setTimeout(60_000);
describe('getDeferredStitchableEntities', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'selects the right rows %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'getDeferredStitchableEntities, %p',
databaseId => {
it('selects the right rows', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -83,6 +83,6 @@ describe('getDeferredStitchableEntities', () => {
expect(+new Date(hitRowAfter!)).toBeGreaterThan(+new Date(hitRowBefore!));
expect(+new Date(missRowAfter!)).toEqual(+new Date(missRowBefore!));
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts
+8 -8
View File
@@ -21,12 +21,12 @@ import { DbStitchQueueRow } from '../../tables';
jest.setTimeout(60_000);
describe('markDeferredStitchCompleted', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'completes only if unchanged %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'markDeferredStitchCompleted, %p',
databaseId => {
it('completes only if unchanged', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -68,6 +68,6 @@ describe('markDeferredStitchCompleted', () => {
stitchTicket: 'the-ticket',
});
await expect(result()).resolves.toEqual([]);
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts
+12 -18
View File
@@ -25,12 +25,12 @@ import {
jest.setTimeout(60_000);
describe('markForStitching', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'marks the right rows in deferred mode %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'markForStitching, %p',
databaseId => {
it('marks the right rows in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -203,12 +203,9 @@ describe('markForStitching', () => {
const final = await result();
const entity4Final = final.find(r => r.entity_ref === 'k:ns/four');
expect(entity4Final?.stitch_ticket).not.toEqual('old');
},
);
});
it.each(databases.eachSupportedId())(
'marks the right rows in immediate mode %p',
async databaseId => {
it('marks the right rows in immediate mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -435,12 +432,9 @@ describe('markForStitching', () => {
for (let i = 0; i < final.length; ++i) {
expect(original[i].next_update_at).not.toEqual(final[i].next_update_at);
}
},
);
});
it.each(databases.eachSupportedId())(
'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p',
async databaseId => {
it('reproduces deadlock scenario when concurrent transactions update overlapping entity sets', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -567,6 +561,6 @@ describe('markForStitching', () => {
expect(row.next_stitch_at).not.toBeNull();
expect(row.stitch_ticket).not.toBeNull();
});
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts
+15 -20
View File
@@ -29,14 +29,15 @@ import { performStitching } from './performStitching';
jest.setTimeout(60_000);
describe('performStitching', () => {
const databases = TestDatabases.create();
const logger = mockServices.logger.mock();
const databases = TestDatabases.create();
// NOTE(freben): Testing the deferred path since it's a superset of the immediate one
it.each(databases.eachSupportedId())(
'runs the happy path in deferred mode for %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'performStitching, %p',
databaseId => {
const logger = mockServices.logger.mock();
// NOTE(freben): Testing the deferred path since it's a superset of the immediate one
it('runs the happy path in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -323,12 +324,9 @@ describe('performStitching', () => {
},
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'handles conflicts with past stitches %p',
async databaseId => {
it('handles conflicts with past stitches', async () => {
if (databaseId === 'MYSQL_8') {
// MySQL doesn't handle conflicts in the same way as the other two, most
// likely due to the conflict probably being handled with a merged even
@@ -391,12 +389,9 @@ describe('performStitching', () => {
'Skipping stitching of k:ns/n, conflict',
expect.anything(),
);
},
);
});
it.each(databases.eachSupportedId())(
'stitches when final_entities row already exists %p',
async databaseId => {
it('stitches when final_entities row already exists', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -444,6 +439,6 @@ describe('performStitching', () => {
expect(entities.length).toBe(1);
expect(entities[0].hash).not.toBe('');
expect(entities[0].final_entity).toBeDefined();
},
);
});
});
},
);
plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts
+105 -105
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Knex } from 'knex';
import { StitchingStrategy } from '../../../stitching/types';
import { applyDatabaseMigrations } from '../../migrations';
@@ -28,109 +28,112 @@ import { deleteOrphanedEntities } from './deleteOrphanedEntities';
jest.setTimeout(60_000);
describe('deleteOrphanedEntities', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
describe.each(databases.eachSupportedId())(
'deleteOrphanedEntities, %p',
databaseId => {
async function createDatabase() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return knex;
}
async function run(knex: Knex, strategy: StitchingStrategy): Promise<number> {
let result: number;
await knex.transaction(
async tx => {
// We can't return here, as knex swallows the return type in case the
// transaction is rolled back:
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
result = await deleteOrphanedEntities({ knex: tx, strategy });
},
{
// If we explicitly trigger a rollback, don't fail.
doNotRejectOnRollback: true,
},
);
return result!;
}
async function run(
knex: Knex,
strategy: StitchingStrategy,
): Promise<number> {
let result: number;
await knex.transaction(
async tx => {
// We can't return here, as knex swallows the return type in case the
// transaction is rolled back:
// https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
result = await deleteOrphanedEntities({ knex: tx, strategy });
},
{
// If we explicitly trigger a rollback, don't fail.
doNotRejectOnRollback: true,
},
);
return result!;
}
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: `id-${ref}`,
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
result_hash: 'original',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: `id-${ref}`,
hash: 'original',
entity_ref: ref,
async function insertEntity(knex: Knex, ...entityRefs: string[]) {
for (const ref of entityRefs) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: `id-${ref}`,
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
result_hash: 'original',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: `id-${ref}`,
hash: 'original',
entity_ref: ref,
});
}
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert(refs);
}
async function insertRelation(knex: Knex, fromRef: string, toRef: string) {
const orig = await knex
.select('entity_id')
.from('refresh_state')
.where('entity_ref', fromRef);
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: orig[0].entity_id,
type: 'fake',
source_entity_ref: fromRef,
target_entity_ref: toRef,
});
}
}
async function insertReference(
knex: Knex,
...refs: DbRefreshStateReferencesRow[]
) {
await knex<DbRefreshStateReferencesRow>('refresh_state_references').insert(
refs,
);
}
async function refreshState(knex: Knex) {
return await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref', 'result_hash');
}
async function insertRelation(knex: Knex, fromRef: string, toRef: string) {
const orig = await knex
.select('entity_id')
.from('refresh_state')
.where('entity_ref', fromRef);
await knex<DbRelationsRow>('relations').insert({
originating_entity_id: orig[0].entity_id,
type: 'fake',
source_entity_ref: fromRef,
target_entity_ref: toRef,
});
}
async function stitchQueue(knex: Knex) {
return await knex('stitch_queue')
.orderBy('entity_ref')
.select('entity_ref');
}
async function refreshState(knex: Knex) {
return await knex<DbRefreshStateRow>('refresh_state')
.orderBy('entity_ref')
.select('entity_ref', 'result_hash');
}
async function finalEntities(knex: Knex) {
return await knex<DbFinalEntitiesRow>('final_entities')
.join(
'refresh_state',
'final_entities.entity_id',
'refresh_state.entity_id',
)
.leftOuterJoin(
'stitch_queue',
'stitch_queue.entity_ref',
'refresh_state.entity_ref',
)
.orderBy('refresh_state.entity_ref')
.select({
entity_ref: 'refresh_state.entity_ref',
hash: 'final_entities.hash',
next_stitch_at: 'stitch_queue.next_stitch_at',
});
}
async function stitchQueue(knex: Knex) {
return await knex('stitch_queue')
.orderBy('entity_ref')
.select('entity_ref');
}
async function finalEntities(knex: Knex) {
return await knex<DbFinalEntitiesRow>('final_entities')
.join(
'refresh_state',
'final_entities.entity_id',
'refresh_state.entity_id',
)
.leftOuterJoin(
'stitch_queue',
'stitch_queue.entity_ref',
'refresh_state.entity_ref',
)
.orderBy('refresh_state.entity_ref')
.select({
entity_ref: 'refresh_state.entity_ref',
hash: 'final_entities.hash',
next_stitch_at: 'stitch_queue.next_stitch_at',
});
}
it.each(databases.eachSupportedId())(
'works for some mixed paths in immediate mode, %p',
async databaseId => {
it('works for some mixed paths in immediate mode', async () => {
/*
In this graph, edges represent refresh state references, not entity relations:
@@ -155,7 +158,7 @@ describe('deleteOrphanedEntities', () => {
Result: E3, E4, E5, E6, and E10 deleted; others remain
Entities that had relations pointing at orphans are marked for reprocessing
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(
knex,
'E1',
@@ -210,12 +213,9 @@ describe('deleteOrphanedEntities', () => {
{ entity_ref: 'E8', hash: 'original', next_stitch_at: null },
{ entity_ref: 'E9', hash: 'original', next_stitch_at: null },
]);
},
);
});
it.each(databases.eachSupportedId())(
'works for some mixed paths in deferred mode, %p',
async databaseId => {
it('works for some mixed paths in deferred mode', async () => {
/*
In this graph, edges represent refresh state references, not entity relations:
@@ -240,7 +240,7 @@ describe('deleteOrphanedEntities', () => {
Result: E3, E4, E5, E6, and E10 deleted; others remain
Entities that had relations pointing at orphans are marked for reprocessing
*/
const knex = await createDatabase(databaseId);
const knex = await createDatabase();
await insertEntity(
knex,
'E1',
@@ -304,6 +304,6 @@ describe('deleteOrphanedEntities', () => {
{ entity_ref: 'E8', hash: 'original', next_stitch_at: null },
{ entity_ref: 'E9', hash: 'original', next_stitch_at: null },
]);
},
);
});
});
},
);
plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts
+123 -171
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TestDatabases } from '@backstage/backend-test-utils';
import {
ANNOTATION_ORIGIN_LOCATION,
stringifyEntityRef,
@@ -37,63 +37,58 @@ import waitFor from 'wait-for-expect';
jest.setTimeout(60_000);
describe('DefaultLocationStore', () => {
const databases = TestDatabases.create();
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
let subscriber: CatalogScmEventsServiceSubscriber | undefined;
const databases = TestDatabases.create();
beforeEach(() => {
jest.clearAllMocks();
describe.each(databases.eachSupportedId())(
'DefaultLocationStore, %p',
databaseId => {
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
let subscriber: CatalogScmEventsServiceSubscriber | undefined;
subscriber = undefined;
mockScmEvents.subscribe.mockImplementation(sub => {
subscriber = sub;
return { unsubscribe: () => {} };
beforeEach(() => {
jest.clearAllMocks();
subscriber = undefined;
mockScmEvents.subscribe.mockImplementation(sub => {
subscriber = sub;
return { unsubscribe: () => {} };
});
});
});
async function createLocationStore(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: true,
unregister: true,
move: true,
});
await store.connect(connection);
return { store, connection, knex };
}
async function createLocationStore() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: true,
unregister: true,
move: true,
});
await store.connect(connection);
return { store, connection, knex };
}
it.each(databases.eachSupportedId())(
'should do a full sync with the locations on connect, %p',
async databaseId => {
const { connection } = await createLocationStore(databaseId);
it('should do a full sync with the locations on connect', async () => {
const { connection } = await createLocationStore();
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
},
);
});
describe('listLocations', () => {
it.each(databases.eachSupportedId())(
'lists empty locations when there is no locations, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('listLocations', () => {
it('lists empty locations when there is no locations', async () => {
const { store } = await createLocationStore();
expect(await store.listLocations()).toEqual([]);
},
);
});
it.each(databases.eachSupportedId())(
'lists locations that are added to the db, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
it('lists locations that are added to the db', async () => {
const { store } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -111,15 +106,12 @@ describe('DefaultLocationStore', () => {
}),
]),
);
},
);
});
});
});
describe('createLocation', () => {
it.each(databases.eachSupportedId())(
'throws when the location already exists, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('createLocation', () => {
it('throws when the location already exists', async () => {
const { store } = await createLocationStore();
const spec = {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -129,13 +121,10 @@ describe('DefaultLocationStore', () => {
await expect(() => store.createLocation(spec)).rejects.toThrow(
new RegExp(`Location ${spec.type}:${spec.target} already exists`),
);
},
);
});
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
@@ -159,13 +148,10 @@ describe('DefaultLocationStore', () => {
},
]),
});
},
);
});
it.each(databases.eachSupportedId())(
'updates refresh_state when onConflict is refresh, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('updates refresh_state when onConflict is refresh', async () => {
const { store, knex } = await createLocationStore();
const spec = {
type: 'url',
target:
@@ -201,13 +187,10 @@ describe('DefaultLocationStore', () => {
expect(new Date(row.next_update_at).getTime()).toBeGreaterThan(
oldDate.getTime(),
);
},
);
});
it.each(databases.eachSupportedId())(
'persists the correct location_entity_ref when creating a location, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('persists the correct location_entity_ref when creating a location', async () => {
const { store, knex } = await createLocationStore();
const created = await store.createLocation({
type: 'url',
target:
@@ -222,26 +205,20 @@ describe('DefaultLocationStore', () => {
expect(row.location_entity_ref).toBe(
'location:default/generated-fa35d9c166e43ab7f4a7c59a00e88e4e8b5aba34',
);
},
);
});
});
});
describe('deleteLocation', () => {
it.each(databases.eachSupportedId())(
'throws if the location does not exist, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('deleteLocation', () => {
it('throws if the location does not exist', async () => {
const { store } = await createLocationStore();
const id = uuid();
await expect(() => store.deleteLocation(id)).rejects.toThrow(
new RegExp(`Found no location with ID ${id}`),
);
},
);
});
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
const location = await store.createLocation({
target:
@@ -268,15 +245,12 @@ describe('DefaultLocationStore', () => {
},
],
});
},
);
});
});
});
describe('updateLocation', () => {
it.each(databases.eachSupportedId())(
'throws if the location does not exist, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
describe('updateLocation', () => {
it('throws if the location does not exist', async () => {
const { store } = await createLocationStore();
const id = uuid();
await expect(() =>
store.updateLocation(id, {
@@ -284,13 +258,10 @@ describe('DefaultLocationStore', () => {
target: 'https://example.com',
}),
).rejects.toThrow(new RegExp(`Found no location with ID ${id}`));
},
);
});
it.each(databases.eachSupportedId())(
'throws ConflictError when updating to a type+target already used by another location, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
it('throws ConflictError when updating to a type+target already used by another location', async () => {
const { store } = await createLocationStore();
await store.createLocation({
type: 'url',
@@ -307,13 +278,10 @@ describe('DefaultLocationStore', () => {
target: 'https://example.com/a',
}),
).rejects.toThrow(/already exists/);
},
);
});
it.each(databases.eachSupportedId())(
'updates type and target and issues a delta mutation with the new entity, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
it('updates type and target and issues a delta mutation with the new entity', async () => {
const { store, connection } = await createLocationStore();
const created = await store.createLocation({
type: 'url',
@@ -345,15 +313,12 @@ describe('DefaultLocationStore', () => {
},
],
});
},
);
});
});
});
describe('getLocationByEntity', () => {
it.each(databases.eachSupportedId())(
'loads correctly, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
describe('getLocationByEntity', () => {
it('loads correctly', async () => {
const { store, knex } = await createLocationStore();
const entityId = uuid();
const locationId = uuid();
@@ -407,16 +372,12 @@ describe('DefaultLocationStore', () => {
).rejects.toMatchInlineSnapshot(
`[NotFoundError: found no entity for ref k:ns/n2]`,
);
},
);
});
});
});
describe('SCM event handling', () => {
describe.each(databases.eachSupportedId())('%p', databaseId => {
describe('SCM event handling', () => {
it('handles location.deleted', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -531,9 +492,7 @@ describe('DefaultLocationStore', () => {
});
it('handles location.moved', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -669,9 +628,7 @@ describe('DefaultLocationStore', () => {
});
it('handles repository.deleted', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -787,9 +744,7 @@ describe('DefaultLocationStore', () => {
});
it('handles repository.moved', async () => {
const { store, knex, connection } = await createLocationStore(
databaseId,
);
const { store, knex, connection } = await createLocationStore();
expect(subscriber).not.toBeUndefined();
// Prepare
@@ -919,45 +874,42 @@ describe('DefaultLocationStore', () => {
});
});
});
});
describe('queryLocations', () => {
const l1 = {
id: '00000000-0000-0000-0000-000000000001',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml',
entityRef:
'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a',
};
const l2 = {
id: '00000000-0000-0000-0000-000000000002',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml',
entityRef:
'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce',
};
const l3 = {
id: '00000000-0000-0000-0000-000000000003',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml',
entityRef:
'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e',
};
const l4 = {
id: '00000000-0000-0000-0000-000000000004',
type: 'file',
target: '/tmp/catalog-info.yaml',
entityRef:
'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2',
};
describe('queryLocations', () => {
const l1 = {
id: '00000000-0000-0000-0000-000000000001',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml',
entityRef:
'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a',
};
const l2 = {
id: '00000000-0000-0000-0000-000000000002',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml',
entityRef:
'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce',
};
const l3 = {
id: '00000000-0000-0000-0000-000000000003',
type: 'url',
target:
'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml',
entityRef:
'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e',
};
const l4 = {
id: '00000000-0000-0000-0000-000000000004',
type: 'file',
target: '/tmp/catalog-info.yaml',
entityRef:
'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2',
};
it.each(databases.eachSupportedId())(
'queries locations correctly, %p',
async databaseId => {
const { store, knex } = await createLocationStore(databaseId);
it('queries locations correctly', async () => {
const { store, knex } = await createLocationStore();
// Insert locations in a random order to test the sorting
const locations = [l1, l2, l3, l4];
@@ -1195,7 +1147,7 @@ describe('DefaultLocationStore', () => {
items: [],
totalItems: 0,
});
},
);
});
});
});
});
},
);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+236 -365
View File
@@ -15,7 +15,6 @@
*/
import {
TestDatabaseId,
TestDatabases,
mockCredentials,
mockServices,
@@ -42,102 +41,103 @@ import { entitiesResponseToObjects } from './response';
jest.setTimeout(60_000);
describe('DefaultEntitiesCatalog', () => {
let knex: Knex;
const databases = TestDatabases.create();
afterEach(async () => {
await knex.destroy();
});
describe.each(databases.eachSupportedId())(
'DefaultEntitiesCatalog, %p',
databaseId => {
let knex: Knex;
const databases = TestDatabases.create();
const stitch = jest.fn();
const stitcher: Stitcher = { stitch } as any;
async function createDatabase(databaseId: TestDatabaseId) {
knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
}
async function addEntity(
entity: Entity,
parents: { source?: string; entity?: Entity }[],
) {
const id = uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
afterEach(async () => {
await knex.destroy();
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
const stitch = jest.fn();
const stitcher: Stitcher = { stitch } as any;
for (const parent of parents) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: parent.source,
source_entity_ref: parent.entity && stringifyEntityRef(parent.entity),
target_entity_ref: stringifyEntityRef(entity),
});
async function createDatabase() {
knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
}
const search = await buildEntitySearch(id, entity);
await knex<DbSearchRow>('search').insert(search);
async function addEntity(
entity: Entity,
parents: { source?: string; entity?: Entity }[],
) {
const id = uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
return id;
}
async function addEntityToSearch(entity: Entity) {
const id = entity.metadata.uid || uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const row of buildEntitySearch(id, entity)) {
await knex<DbSearchRow>('search').insert({
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
key: row.key,
value: row.value,
original_value: row.original_value,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const parent of parents) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: parent.source,
source_entity_ref: parent.entity && stringifyEntityRef(parent.entity),
target_entity_ref: stringifyEntityRef(entity),
});
}
const search = await buildEntitySearch(id, entity);
await knex<DbSearchRow>('search').insert(search);
return id;
}
}
afterEach(() => {
jest.resetAllMocks();
});
async function addEntityToSearch(entity: Entity) {
const id = entity.metadata.uid || uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
describe('entityAncestry', () => {
it.each(databases.eachSupportedId())(
'should return the ancestry with one parent, %p',
async databaseId => {
await createDatabase(databaseId);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
entity_ref: entityRef,
final_entity: entityJson,
hash: 'h',
});
for (const row of buildEntitySearch(id, entity)) {
await knex<DbSearchRow>('search').insert({
entity_id: id,
key: row.key,
value: row.value,
original_value: row.original_value,
});
}
}
afterEach(() => {
jest.resetAllMocks();
});
describe('entityAncestry', () => {
it('should return the ancestry with one parent', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -188,13 +188,10 @@ describe('DefaultEntitiesCatalog', () => {
},
]),
);
},
);
});
it.each(databases.eachSupportedId())(
'should throw error if the entity does not exist, %p',
async databaseId => {
await createDatabase(databaseId);
it('should throw error if the entity does not exist', async () => {
await createDatabase();
const catalog = new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
@@ -203,13 +200,10 @@ describe('DefaultEntitiesCatalog', () => {
await expect(() =>
catalog.entityAncestry('k:default/root'),
).rejects.toThrow('No such entity k:default/root');
},
);
});
it.each(databases.eachSupportedId())(
'should return the ancestry with multiple parents, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return the ancestry with multiple parents', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -275,15 +269,12 @@ describe('DefaultEntitiesCatalog', () => {
},
]),
);
},
);
});
});
});
describe('entities', () => {
it.each(databases.eachSupportedId())(
'should return correct entity for simple filter, %p',
async databaseId => {
await createDatabase(databaseId);
describe('entities', () => {
it('should return correct entity for simple filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -317,13 +308,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity2);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entity for negation filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entity for negation filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -359,13 +347,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity1);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entities for nested filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entities for nested filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -433,13 +418,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(2);
expect(entities).toContainEqual(entity2);
expect(entities).toContainEqual(entity4);
},
);
});
it.each(databases.eachSupportedId())(
'should return correct entities for complex negation filter, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return correct entities for complex negation filter', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -480,14 +462,11 @@ describe('DefaultEntitiesCatalog', () => {
expect(entities.length).toBe(1);
expect(entities).toContainEqual(entity1);
},
);
});
it.each(databases.eachSupportedId())(
'should return no matches for an empty values array, %p',
// NOTE: An empty values array is not a sensible input in a realistic scenario.
async databaseId => {
await createDatabase(databaseId);
it('should return no matches for an empty values array', async () => {
// NOTE: An empty values array is not a sensible input in a realistic scenario.
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
@@ -519,13 +498,10 @@ describe('DefaultEntitiesCatalog', () => {
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(0);
},
);
});
it.each(databases.eachSupportedId())(
'should return both target and targetRef for entities in compat mode',
async databaseId => {
await createDatabase(databaseId);
it('should return both target and targetRef for entities in compat mode', async () => {
await createDatabase();
await addEntity(
{
apiVersion: 'a',
@@ -579,13 +555,10 @@ describe('DefaultEntitiesCatalog', () => {
target: { kind: 'x', namespace: 'y', name: 'z' },
},
]);
},
);
});
it.each(databases.eachSupportedId())(
'handles inversion both for existing and missing keys, %p',
async databaseId => {
await createDatabase(databaseId);
it('handles inversion both for existing and missing keys', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
@@ -638,13 +611,10 @@ describe('DefaultEntitiesCatalog', () => {
filter: { not: { key: 'spec.b', values: ['lonely'] } },
}),
).resolves.toEqual(['n1', 'n3']);
},
);
});
it.each(databases.eachSupportedId())(
'can order and combine with filtering, %p',
async databaseId => {
await createDatabase(databaseId);
it('can order and combine with filtering', async () => {
await createDatabase();
const entity1: Entity = {
apiVersion: 'a',
@@ -739,13 +709,10 @@ describe('DefaultEntitiesCatalog', () => {
],
}),
).resolves.toEqual(['n4', 'n3', 'n1', 'n2']);
},
);
});
it.each(databases.eachSupportedId())(
'paginates correctly through single-field ordering, %p',
async databaseId => {
await createDatabase(databaseId);
it('paginates correctly through single-field ordering', async () => {
await createDatabase();
// All four entities have metadata.name — fast path uses Phase 1 only
for (const name of ['n1', 'n2', 'n3', 'n4']) {
@@ -795,13 +762,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(await hasNext(2, 1)).toBe(true);
await expect(page(100)).resolves.toEqual(['n1', 'n2', 'n3', 'n4']);
},
);
});
it.each(databases.eachSupportedId())(
'paginates across the Phase 1 / Phase 2 boundary, %p',
async databaseId => {
await createDatabase(databaseId);
it('paginates across the Phase 1 / Phase 2 boundary', async () => {
await createDatabase();
// n1 and n2 have spec.b (Phase 1); n3 and n4 do not (Phase 2).
// Explicit UIDs pin Phase 2 ordering (entity_id ASC) to a known sequence.
@@ -867,13 +831,10 @@ describe('DefaultEntitiesCatalog', () => {
'n3',
'n4',
]);
},
);
});
it.each(databases.eachSupportedId())(
'treats a null sort-field value the same as a missing sort field, %p',
async databaseId => {
await createDatabase(databaseId);
it('treats a null sort-field value the same as a missing sort field', async () => {
await createDatabase();
// n1 has spec.b with a real value (Phase 1)
// n2 has spec.b explicitly set to null — buildEntitySearch stores value=NULL
@@ -918,15 +879,12 @@ describe('DefaultEntitiesCatalog', () => {
// ordered by entity_id ASC, regardless of primary direction
await expect(page('asc')).resolves.toEqual(['n1', 'n2', 'n3']);
await expect(page('desc')).resolves.toEqual(['n1', 'n2', 'n3']);
},
);
});
});
});
describe('entitiesBatch', () => {
it.each(databases.eachSupportedId())(
'queries for entities by ref, including duplicates, and gracefully returns null for missing entities, %p',
async databaseId => {
await createDatabase(databaseId);
describe('entitiesBatch', () => {
it('queries for entities by ref, including duplicates, and gracefully returns null for missing entities', async () => {
await createDatabase();
await addEntity(
{
@@ -976,13 +934,10 @@ describe('DefaultEntitiesCatalog', () => {
null,
'k:default/two',
]);
},
);
});
it.each(databases.eachSupportedId())(
'queries for entities by ref, including filtering, %p',
async databaseId => {
await createDatabase(databaseId);
it('queries for entities by ref, including filtering', async () => {
await createDatabase();
await addEntity(
{
@@ -1022,15 +977,12 @@ describe('DefaultEntitiesCatalog', () => {
'k:default/two',
null,
]);
},
);
});
});
});
describe('queryEntities', () => {
it.each(databases.eachSupportedId())(
'should return paginated entities and scroll the items accordingly, %p',
async databaseId => {
await createDatabase(databaseId);
describe('queryEntities', () => {
it('should return paginated entities and scroll the items accordingly', async () => {
await createDatabase();
const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1201,13 +1153,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
},
);
});
it.each(databases.eachSupportedId())(
'should return paginated entities ordered in descending order and scroll the items accordingly, %p',
async databaseId => {
await createDatabase(databaseId);
it('should return paginated entities ordered in descending order and scroll the items accordingly', async () => {
await createDatabase();
const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1379,13 +1328,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the results when query is provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the results when query is provided', async () => {
await createDatabase();
const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
const entities: Entity[] = names.map(name => entityFrom(name));
@@ -1435,13 +1381,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response.pageInfo.nextCursor).toBeUndefined();
expect(response.pageInfo.prevCursor).toBeUndefined();
expect(response.totalItems).toBe(3);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the results when query is provided with fullTextFilter for camelCase fields, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the results when query is provided with fullTextFilter for camelCase fields', async () => {
await createDatabase();
const entities: Entity[] = [
{
@@ -1489,13 +1432,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response.pageInfo.nextCursor).toBeUndefined();
expect(response.pageInfo.prevCursor).toBeUndefined();
expect(response.totalItems).toBe(1);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the text results when sortOrder is not provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the text results when sortOrder is not provided', async () => {
await createDatabase();
const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's'];
const entities: Entity[] = names.map((name, index) =>
@@ -1577,13 +1517,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
},
);
});
it.each(databases.eachSupportedId())(
'should filter the text results by multiple search fields if provided, %p',
async databaseId => {
await createDatabase(databaseId);
it('should filter the text results by multiple search fields if provided', async () => {
await createDatabase();
const defs = [
{
@@ -1682,13 +1619,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
expect(paginatedResponsePrev).toMatchObject(paginatedResponse);
},
);
});
it.each(databases.eachSupportedId())(
'should include totalItems and empty entities in the response in case limit is zero, %p',
async databaseId => {
await createDatabase(databaseId);
it('should include totalItems and empty entities in the response in case limit is zero', async () => {
await createDatabase();
await Promise.all(
Array(20)
@@ -1718,13 +1652,10 @@ describe('DefaultEntitiesCatalog', () => {
items: { type: 'raw', entities: [] },
pageInfo: {},
});
},
);
});
it.each(databases.eachSupportedId())(
'can skip totalItems, %p',
async databaseId => {
await createDatabase(databaseId);
it('can skip totalItems', async () => {
await createDatabase();
await Promise.all(
Array(15)
@@ -1772,13 +1703,10 @@ describe('DefaultEntitiesCatalog', () => {
pageInfo: { prevCursor: expect.anything() },
});
expect(response.items.entities).toHaveLength(5);
},
);
});
it.each(databases.eachSupportedId())(
'should paginate results accordingly in case of clashing items, %p',
async databaseId => {
await createDatabase(databaseId);
it('should paginate results accordingly in case of clashing items', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA')),
@@ -1871,13 +1799,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should exclude filtered entities when paginating, %p',
async databaseId => {
await createDatabase(databaseId);
it('should exclude filtered entities when paginating', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: '1', kind: 'included' })),
@@ -1954,13 +1879,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response2.pageInfo.nextCursor).toBeDefined();
expect(response2.pageInfo.prevCursor).toBeDefined();
expect(response2.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should paginate results without sort fields, %p',
async databaseId => {
await createDatabase(databaseId);
it('should paginate results without sort fields', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: 'id1' })),
@@ -2058,13 +1980,10 @@ describe('DefaultEntitiesCatalog', () => {
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(6);
},
);
});
it.each(databases.eachSupportedId())(
'should sort properly for fields that do not exist on all entities, %p',
async databaseId => {
await createDatabase(databaseId);
it('should sort properly for fields that do not exist on all entities', async () => {
await createDatabase();
await Promise.all([
addEntityToSearch(entityFrom('AA', { uid: 'id1' })),
@@ -2100,13 +2019,10 @@ describe('DefaultEntitiesCatalog', () => {
),
).toEqual(['BB', 'CC']);
expect(descResult.totalItems).toBe(2);
},
);
});
it.each(databases.eachSupportedId())(
'should silently skip over entities that are not yet stitched, %p',
async databaseId => {
await createDatabase(databaseId);
it('should silently skip over entities that are not yet stitched', async () => {
await createDatabase();
const entity1 = entityFrom('AA', { uid: 'id1' });
const entity2 = entityFrom('BB', { uid: 'id2' });
@@ -2149,13 +2065,10 @@ describe('DefaultEntitiesCatalog', () => {
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB']);
},
);
});
it.each(databases.eachSupportedId())(
'should not return duplicate entities when using orderField, %p',
async databaseId => {
await createDatabase(databaseId);
it('should not return duplicate entities when using orderField', async () => {
await createDatabase();
// Create a few test entities with different names to sort by
const entities = [
@@ -2234,13 +2147,10 @@ describe('DefaultEntitiesCatalog', () => {
'b-entity',
'c-entity',
]);
},
);
});
it.each(databases.eachSupportedId())(
'should apply both filter and query when both are given, %p',
async databaseId => {
await createDatabase(databaseId);
it('should apply both filter and query when both are given', async () => {
await createDatabase();
// Add entities with different kinds and names
await addEntityToSearch(entityFrom('A', { kind: 'component' }));
@@ -2266,15 +2176,12 @@ describe('DefaultEntitiesCatalog', () => {
expect(resultEntities).toEqual([
entityFrom('A', { kind: 'component' }),
]);
},
);
});
});
});
describe('removeEntityByUid', () => {
it.each(databases.eachSupportedId())(
'also clears parent hashes, %p',
async databaseId => {
await createDatabase(databaseId);
describe('removeEntityByUid', () => {
it('also clears parent hashes', async () => {
await createDatabase();
const grandparent: Entity = {
apiVersion: 'a',
@@ -2358,15 +2265,12 @@ describe('DefaultEntitiesCatalog', () => {
expect(stitch).toHaveBeenCalledWith({
entityRefs: new Set(['k:default/unrelated1', 'k:default/unrelated2']),
});
},
);
});
});
});
describe('facets', () => {
it.each(databases.eachSupportedId())(
'can filter and collect properly, %p',
async databaseId => {
await createDatabase(databaseId);
describe('facets', () => {
it('can filter and collect properly', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2432,13 +2336,10 @@ describe('DefaultEntitiesCatalog', () => {
kind: [{ value: 'k', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'can match on annotations and labels with dots in them, %p',
async databaseId => {
await createDatabase(databaseId);
it('can match on annotations and labels with dots in them', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2483,13 +2384,10 @@ describe('DefaultEntitiesCatalog', () => {
],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'can match on strings in arrays, %p',
async databaseId => {
await createDatabase(databaseId);
it('can match on strings in arrays', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2529,13 +2427,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'works with a mixture of present and missing facets, %p',
async databaseId => {
await createDatabase(databaseId);
it('works with a mixture of present and missing facets', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2573,13 +2468,10 @@ describe('DefaultEntitiesCatalog', () => {
missing: [],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'works when the entity is duplicated in search results, %p',
async databaseId => {
await createDatabase(databaseId);
it('works when the entity is duplicated in search results', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2621,28 +2513,22 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'one', count: 1 }],
},
});
},
);
async function setupFacetsCatalog(
databaseId: TestDatabaseId,
entities: Entity[],
) {
await createDatabase(databaseId);
for (const entity of entities) {
await addEntityToSearch(entity);
}
return new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
stitcher,
});
}
it.each(databases.eachSupportedId())(
'excludes not-yet-stitched entities from filtered facets, %p',
async databaseId => {
await createDatabase(databaseId);
async function setupFacetsCatalog(entities: Entity[]) {
await createDatabase();
for (const entity of entities) {
await addEntityToSearch(entity);
}
return new DefaultEntitiesCatalog({
database: knex,
logger: mockServices.logger.mock(),
stitcher,
});
}
it('excludes not-yet-stitched entities from filtered facets', async () => {
await createDatabase();
await addEntityToSearch({
apiVersion: 'a',
@@ -2701,13 +2587,10 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'stitched', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with a predicate query, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with a predicate query', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2742,13 +2625,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with a predicate query using $in, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with a predicate query using $in', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2783,13 +2663,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with compound allOf filter, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with compound allOf filter', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2826,13 +2703,10 @@ describe('DefaultEntitiesCatalog', () => {
'metadata.name': [{ value: 'one', count: 1 }],
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with compound anyOf filter, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with compound anyOf filter', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2872,13 +2746,10 @@ describe('DefaultEntitiesCatalog', () => {
]),
},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters with both filter and query combined, %p',
async databaseId => {
const catalog = await setupFacetsCatalog(databaseId, [
it('filters with both filter and query combined', async () => {
const catalog = await setupFacetsCatalog([
{
apiVersion: 'a',
kind: 'Component',
@@ -2911,10 +2782,10 @@ describe('DefaultEntitiesCatalog', () => {
'spec.type': [{ value: 'service', count: 1 }],
},
});
},
);
});
});
});
});
},
);
function entityFrom(
name: string,
plugins/catalog-backend/src/service/DefaultRefreshService.test.ts
+152 -167
View File
@@ -17,7 +17,6 @@
import {
mockCredentials,
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
@@ -42,166 +41,162 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('DefaultRefreshService', () => {
const defaultLogger = mockServices.logger.mock();
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createDatabase(
databaseId: TestDatabaseId,
logger: LoggerService = defaultLogger,
) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
processingDb: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: () => 100,
events: mockServices.events.mock(),
describe.each(databases.eachSupportedId())(
'DefaultRefreshService, %p',
databaseId => {
const defaultLogger = mockServices.logger.mock();
async function createDatabase(logger: LoggerService = defaultLogger) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
return {
knex,
processingDb: new DefaultProcessingDatabase({
database: knex,
logger,
refreshInterval: () => 100,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
}),
catalogDb: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
const createPopulatedEngine = async (options: {
db: ProcessingDatabase;
knex: Knex;
entities: Entity[];
references: { [source: string]: string[] };
entityProcessor?: (entity: Entity) => void;
}) => {
const { db, knex, entities, references, entityProcessor } = options;
const entityMap = new Map(
entities.map(entity => [stringifyEntityRef(entity), entity]),
);
for (const entity of entities) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: stringifyEntityRef(entity),
unprocessed_entity: JSON.stringify(entity),
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
const entitiesWithParent = new Set(Object.values(references).flat());
for (const entityRef of entityMap.keys()) {
if (!entitiesWithParent.has(entityRef)) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: 'ConfigLocationProvider',
target_entity_ref: entityRef,
});
}
}
for (const [sourceRef, targetRefs] of Object.entries(references)) {
for (const targetRef of targetRefs) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_entity_ref: sourceRef,
target_entity_ref: targetRef,
});
}
}
const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), {
knex,
logger: defaultLogger,
metrics: metricsServiceMock.mock(),
}),
catalogDb: new DefaultCatalogDatabase({
database: knex,
logger,
}),
};
}
const createPopulatedEngine = async (options: {
db: ProcessingDatabase;
knex: Knex;
entities: Entity[];
references: { [source: string]: string[] };
entityProcessor?: (entity: Entity) => void;
}) => {
const { db, knex, entities, references, entityProcessor } = options;
const entityMap = new Map(
entities.map(entity => [stringifyEntityRef(entity), entity]),
);
for (const entity of entities) {
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: uuid(),
entity_ref: stringifyEntityRef(entity),
unprocessed_entity: JSON.stringify(entity),
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
const engine = new DefaultCatalogProcessingEngine({
config: new ConfigReader({}),
logger: defaultLogger,
processingDatabase: db,
knex: knex,
stitcher: stitcher,
scheduler: mockServices.scheduler(),
orchestrator: {
async process(request: EntityProcessingRequest) {
const entityRef = stringifyEntityRef(request.entity);
const entity = entityMap.get(entityRef);
if (!entity) {
throw new Error(`Unexpected entity: ${entityRef}`);
}
const deferredEntities =
references[entityRef]?.map(ref => {
const e = entityMap.get(ref);
if (!e) {
throw new Error(`Target entity not found: ${ref}`);
}
return { entity: e, locationKey: ref };
}) || [];
const entitiesWithParent = new Set(Object.values(references).flat());
for (const entityRef of entityMap.keys()) {
if (!entitiesWithParent.has(entityRef)) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_key: 'ConfigLocationProvider',
target_entity_ref: entityRef,
});
}
}
for (const [sourceRef, targetRefs] of Object.entries(references)) {
for (const targetRef of targetRefs) {
await knex<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert({
source_entity_ref: sourceRef,
target_entity_ref: targetRef,
});
}
}
entityProcessor?.(entity);
const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), {
knex,
logger: defaultLogger,
metrics: metricsServiceMock.mock(),
});
const engine = new DefaultCatalogProcessingEngine({
config: new ConfigReader({}),
logger: defaultLogger,
processingDatabase: db,
knex: knex,
stitcher: stitcher,
scheduler: mockServices.scheduler(),
orchestrator: {
async process(request: EntityProcessingRequest) {
const entityRef = stringifyEntityRef(request.entity);
const entity = entityMap.get(entityRef);
if (!entity) {
throw new Error(`Unexpected entity: ${entityRef}`);
}
const deferredEntities =
references[entityRef]?.map(ref => {
const e = entityMap.get(ref);
if (!e) {
throw new Error(`Target entity not found: ${ref}`);
}
return { entity: e, locationKey: ref };
}) || [];
entityProcessor?.(entity);
return {
ok: true,
completedEntity: {
...entity,
metadata: {
...entity.metadata,
annotations: {
...entity.metadata.annotations,
'refresh-completed': 'true',
return {
ok: true,
completedEntity: {
...entity,
metadata: {
...entity.metadata,
annotations: {
...entity.metadata.annotations,
'refresh-completed': 'true',
},
},
},
},
relations: [],
errors: [],
deferredEntities,
state: {},
refreshKeys: [],
};
relations: [],
errors: [],
deferredEntities,
state: {},
refreshKeys: [],
};
},
},
},
createHash: () => createHash('sha1'),
pollingIntervalMs: 50,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
});
createHash: () => createHash('sha1'),
pollingIntervalMs: 50,
events: mockServices.events.mock(),
metrics: metricsServiceMock.mock(),
});
return engine;
};
return engine;
};
const waitForRefresh = async (knex: Knex, entityRef: string) => {
for (;;) {
const [result] = await knex<DbRefreshStateRow>('refresh_state')
.where('entity_ref', entityRef)
.select();
const waitForRefresh = async (knex: Knex, entityRef: string) => {
for (;;) {
const [result] = await knex<DbRefreshStateRow>('refresh_state')
.where('entity_ref', entityRef)
.select();
const entity = result.processed_entity
? (JSON.parse(result.processed_entity) as Entity)
: undefined;
if (entity?.metadata?.annotations?.['refresh-completed']) {
// Reset the annotation so that we can run another verification
delete entity.metadata.annotations['refresh-completed'];
await knex<DbRefreshStateRow>('refresh_state')
.update({
processed_entity: JSON.stringify(entity),
})
.where('entity_ref', entityRef);
return true;
const entity = result.processed_entity
? (JSON.parse(result.processed_entity) as Entity)
: undefined;
if (entity?.metadata?.annotations?.['refresh-completed']) {
// Reset the annotation so that we can run another verification
delete entity.metadata.annotations['refresh-completed'];
await knex<DbRefreshStateRow>('refresh_state')
.update({
processed_entity: JSON.stringify(entity),
})
.where('entity_ref', entityRef);
return true;
}
await new Promise(resolve => setTimeout(resolve, 500));
}
await new Promise(resolve => setTimeout(resolve, 500));
}
};
};
it.each(databases.eachSupportedId())(
'should refresh the parent location, %p',
async databaseId => {
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
it('should refresh the parent location', async () => {
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -239,15 +234,10 @@ describe('DefaultRefreshService', () => {
).resolves.toBe(true);
await engine.stop();
},
);
});
it.each(databases.eachSupportedId())(
'should refresh the location further up the tree, %p',
async databaseId => {
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
it('should refresh the location further up the tree', async () => {
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -293,16 +283,11 @@ describe('DefaultRefreshService', () => {
);
await engine.stop();
},
);
});
it.each(databases.eachSupportedId())(
'should refresh even when parent has no changes',
async databaseId => {
it('should refresh even when parent has no changes', async () => {
let secondRound = false;
const { knex, processingDb, catalogDb } = await createDatabase(
databaseId,
);
const { knex, processingDb, catalogDb } = await createDatabase();
const refreshService = new DefaultRefreshService({ database: catalogDb });
const engine = await createPopulatedEngine({
db: processingDb,
@@ -356,6 +341,6 @@ describe('DefaultRefreshService', () => {
).resolves.toBe(true);
await engine.stop();
},
);
});
});
},
);
plugins/catalog-backend/src/service/createRouter.test.ts
+51 -55
View File
@@ -19,7 +19,6 @@ import { wrapServer } from '@backstage/backend-openapi-utils/testUtils';
import {
mockCredentials,
mockServices,
TestDatabaseId,
TestDatabases,
} from '@backstage/backend-test-utils';
import type { Location } from '@backstage/catalog-client';
@@ -1680,59 +1679,59 @@ describe('createRouter readonly and raw json enabled', () => {
});
});
describe('POST /locations/by-query works end to end', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
async function createTestRouter(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
describe.each(databases.eachSupportedId())(
'POST /locations/by-query works end to end, %p',
databaseId => {
async function createTestRouter() {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
const mockScmEvents = {
subscribe: jest.fn(),
publish: jest.fn(),
markEventActionTaken: jest.fn(),
};
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: false,
unregister: false,
move: false,
});
await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() });
const store = new DefaultLocationStore(knex, mockScmEvents, {
refresh: false,
unregister: false,
move: false,
});
await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() });
const locationService = new DefaultLocationService(
store,
{ process: jest.fn() },
{
allowedLocationTypes: ['url'],
defaultLocationConflictStrategy: 'reject',
},
);
const locationService = new DefaultLocationService(
store,
{ process: jest.fn() },
{
allowedLocationTypes: ['url'],
defaultLocationConflictStrategy: 'reject',
},
);
const router = await createRouter({
locationService,
logger: mockServices.logger.mock(),
config: new ConfigReader(undefined),
auth: mockServices.auth(),
httpAuth: mockServices.httpAuth(),
orchestrator: { process: jest.fn() },
permissionsService: mockServices.permissions(),
auditor: mockServices.auditor.mock(),
});
const router = await createRouter({
locationService,
logger: mockServices.logger.mock(),
config: new ConfigReader(undefined),
auth: mockServices.auth(),
httpAuth: mockServices.httpAuth(),
orchestrator: { process: jest.fn() },
permissionsService: mockServices.permissions(),
auditor: mockServices.auditor.mock(),
});
const errorMiddleware = MiddlewareFactory.create({
logger: mockServices.logger.mock(),
config: mockServices.rootConfig(),
});
router.use(errorMiddleware.error());
const errorMiddleware = MiddlewareFactory.create({
logger: mockServices.logger.mock(),
config: mockServices.rootConfig(),
});
router.use(errorMiddleware.error());
return { knex, app: express().use(router) };
}
return { knex, app: express().use(router) };
}
it.each(databases.eachSupportedId())(
'paginates through locations correctly, %p',
async databaseId => {
const { knex, app } = await createTestRouter(databaseId);
it('paginates through locations correctly', async () => {
const { knex, app } = await createTestRouter();
// Insert 5 locations directly into the database
const locations = [
@@ -1819,13 +1818,10 @@ describe('POST /locations/by-query works end to end', () => {
totalItems: 5,
pageInfo: {},
});
},
);
});
it.each(databases.eachSupportedId())(
'filters locations with query parameter, %p',
async databaseId => {
const { knex, app } = await createTestRouter(databaseId);
it('filters locations with query parameter', async () => {
const { knex, app } = await createTestRouter();
// Insert locations with different types
const locations = [
@@ -1874,9 +1870,9 @@ describe('POST /locations/by-query works end to end', () => {
totalItems: 2,
pageInfo: {},
});
},
);
});
});
},
);
function mockCursor(partialCursor?: Partial<Cursor>): Cursor {
return {
plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts
+226 -228
View File
@@ -29,248 +29,246 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha';
jest.setTimeout(60_000);
describe('Stitcher', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
describe.each(databases.eachSupportedId())('Stitcher, %p', databaseId => {
const logger = mockServices.logger.mock();
it.each(databases.eachSupportedId())(
'runs the happy path for %p',
async databaseId => {
const db = await databases.init(databaseId);
await applyDatabaseMigrations(db);
it('runs the happy path', async () => {
const db = await databases.init(databaseId);
await applyDatabaseMigrations(db);
const stitcher = new DefaultStitcher({
knex: db,
logger,
strategy: { mode: 'immediate' },
metrics: metricsServiceMock.mock(),
});
let entities: DbFinalEntitiesRow[];
let entity: Entity;
const stitcher = new DefaultStitcher({
knex: db,
logger,
strategy: { mode: 'immediate' },
metrics: metricsServiceMock.mock(),
});
let entities: DbFinalEntitiesRow[];
let entity: Entity;
await db<DbRefreshStateRow>('refresh_state').insert([
await db<DbRefreshStateRow>('refresh_state').insert([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
},
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
// handles and ignores duplicates
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const last_updated_at = entities[0].last_updated_at;
expect(last_updated_at).not.toBeNull();
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
// handles and ignores duplicates
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const last_updated_at = entities[0].last_updated_at;
expect(last_updated_at).not.toBeNull();
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
// Re-stitch without any changes
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
// Re-stitch without any changes
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/third',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
target_entity_ref: 'k:ns/third',
targetRef: 'k:ns/other',
},
]);
await stitcher.stitch({ entityRefs: ['k:ns/n'] });
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
targetRef: 'k:ns/other',
},
{
type: 'looksAt',
targetRef: 'k:ns/third',
},
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
{
type: 'looksAt',
targetRef: 'k:ns/third',
},
spec: {
k: 'v',
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/other',
value: 'k:ns/other',
},
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/third',
value: 'k:ns/third',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
},
);
{
entity_id: 'my-id',
key: 'relations.looksat',
original_value: 'k:ns/third',
value: 'k:ns/third',
},
{
entity_id: 'my-id',
key: 'apiversion',
original_value: 'a',
value: 'a',
},
{
entity_id: 'my-id',
key: 'kind',
original_value: 'k',
value: 'k',
},
{
entity_id: 'my-id',
key: 'metadata.name',
original_value: 'n',
value: 'n',
},
{
entity_id: 'my-id',
key: 'metadata.namespace',
original_value: 'ns',
value: 'ns',
},
{
entity_id: 'my-id',
key: 'metadata.uid',
original_value: 'my-id',
value: 'my-id',
},
{
entity_id: 'my-id',
key: 'spec.k',
original_value: 'v',
value: 'v',
},
]),
);
});
});
plugins/catalog-backend/src/tests/migrations.test.ts
+1055 -1105
View File
File diff suppressed because it is too large Load Diff
plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts
+10 -14
View File
@@ -144,14 +144,13 @@ class Tracker {
}
}
describePerformanceTest('stitchingPerformance', () => {
const databases = TestDatabases.create({
ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
const databases = TestDatabases.create({
ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
'runs stitching in immediate mode, %p',
async databaseId => {
describePerformanceTest('stitchingPerformance', () => {
describe.each(databases.eachSupportedId())('%p', databaseId => {
it('runs stitching in immediate mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -199,12 +198,9 @@ describePerformanceTest('stitchingPerformance', () => {
await expect(tracker.completion()).resolves.toBeUndefined();
await backend.stop();
await knex.destroy();
},
);
});
it.each(databases.eachSupportedId())(
'runs stitching in deferred mode, %p',
async databaseId => {
it('runs stitching in deferred mode', async () => {
const knex = await databases.init(databaseId);
await applyDatabaseMigrations(knex);
@@ -252,6 +248,6 @@ describePerformanceTest('stitchingPerformance', () => {
await expect(tracker.completion()).resolves.toBeUndefined();
await backend.stop();
await knex.destroy();
},
);
});
});
});
plugins/events-backend/src/migrations.test.ts
+37 -43
View File
@@ -45,60 +45,54 @@ const databases = TestDatabases.create({
ids: ['POSTGRES_9', 'POSTGRES_14', 'POSTGRES_16'],
});
const maybeDescribe =
databases.eachSupportedId().length > 0 ? describe : describe.skip;
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20240523100528_init.js', async () => {
const knex = await databases.init(databaseId);
maybeDescribe('migrations', () => {
it.each(databases.eachSupportedId())(
'20240523100528_init.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20240523100528_init.js');
await migrateUpOnce(knex);
await migrateUntilBefore(knex, '20240523100528_init.js');
await migrateUpOnce(knex);
await knex('event_bus_events').insert({
topic: 'test',
created_by: 'abc',
data_json: JSON.stringify({ message: 'hello' }),
notified_subscribers: ['tester'],
});
await knex('event_bus_subscriptions').insert({
id: 'tester',
created_by: 'abc',
read_until: '5',
topics: ['test', 'test2'],
});
await knex('event_bus_events').insert({
topic: 'test',
await expect(knex('event_bus_events')).resolves.toEqual([
{
id: '1',
created_by: 'abc',
topic: 'test',
data_json: JSON.stringify({ message: 'hello' }),
created_at: expect.anything(),
notified_subscribers: ['tester'],
});
await knex('event_bus_subscriptions').insert({
},
]);
await expect(knex('event_bus_subscriptions')).resolves.toEqual([
{
id: 'tester',
created_by: 'abc',
created_at: expect.anything(),
updated_at: expect.anything(),
read_until: '5',
topics: ['test', 'test2'],
});
},
]);
await expect(knex('event_bus_events')).resolves.toEqual([
{
id: '1',
created_by: 'abc',
topic: 'test',
data_json: JSON.stringify({ message: 'hello' }),
created_at: expect.anything(),
notified_subscribers: ['tester'],
},
]);
await expect(knex('event_bus_subscriptions')).resolves.toEqual([
{
id: 'tester',
created_by: 'abc',
created_at: expect.anything(),
updated_at: expect.anything(),
read_until: '5',
topics: ['test', 'test2'],
},
]);
await migrateDownOnce(knex);
await migrateDownOnce(knex);
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('event_bus_events')).rejects.toEqual(expect.anything());
// This looks odd - you might expect a .toThrow at the end but that
// actually is flaky for some reason specifically on sqlite when
// performing multiple runs in sequence
await expect(knex('event_bus_events')).rejects.toEqual(expect.anything());
await knex.destroy();
},
);
await knex.destroy();
});
});
plugins/events-backend/src/service/EventsPlugin.test.ts
+236 -258
View File
@@ -22,7 +22,6 @@ import {
} from '@backstage/backend-plugin-api';
import {
TestBackend,
TestDatabaseId,
TestDatabases,
mockCredentials,
mockServices,
@@ -105,8 +104,15 @@ describe('eventsPlugin', () => {
test: 'fake-ext',
});
});
});
describe('event bus', () => {
const eventBusDatabases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'],
});
describe.each(eventBusDatabases.eachSupportedId())(
'event bus, %p',
databaseId => {
class ReqHelper {
private readonly backend: TestBackend;
@@ -146,12 +152,8 @@ describe('eventsPlugin', () => {
}
}
const databases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'],
});
async function mockKnexFactory(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
async function mockKnexFactory() {
const knex = await eventBusDatabases.init(databaseId);
return mockServices.database.factory({ knex });
}
@@ -163,278 +165,254 @@ describe('eventsPlugin', () => {
}
});
it.each(databases.eachSupportedId())(
'should be possible to publish events as a service, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should be possible to publish events as a service', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.service.header())
.expect(204); // 204, since there are no subscribers
},
);
await helper
.publish('test', { n: 1 })
.set('authorization', mockCredentials.service.header())
.expect(204); // 204, since there are no subscribers
});
it.each(databases.eachSupportedId())(
'should be possible to subscribe as a service and receive an event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should be possible to subscribe as a service and receive an event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.subscribe('tester', ['test'])
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper.subscribe('tester', ['test']).expect(201);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.none.header())
.expect(401);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper
.readEvents('tester')
.set('authorization', mockCredentials.user.header())
.expect(403);
await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber
await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber
await helper.readEvents('tester').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
},
);
await helper.readEvents('tester').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
});
it.each(databases.eachSupportedId())(
'should only send an event for each subscriber once, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
it('should only send an event for each subscriber once', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// A single event
await helper.publish('test', { n: 1 }).expect(201);
// A single event
await helper.publish('test', { n: 1 }).expect(201);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Two clients for subscriber 2, only one gets the event
const res1 = helper.readEvents('tester-2');
const res2 = helper.readEvents('tester-2');
// Two clients for subscriber 2, only one gets the event
const res1 = helper.readEvents('tester-2');
const res2 = helper.readEvents('tester-2');
const res = await Promise.race([res1, res2]);
expect(res.status).toBe(200);
expect(res.body).toEqual({
events: [{ topic: 'test', payload: { n: 1 } }],
});
const res = await Promise.race([res1, res2]);
expect(res.status).toBe(200);
expect(res.body).toEqual({
events: [{ topic: 'test', payload: { n: 1 } }],
});
// Post another event, which triggers the other client to return
await helper.publish('test', { n: 2 }).expect(201);
// Post another event, which triggers the other client to return
await helper.publish('test', { n: 2 }).expect(201);
const otherRes = await Promise.all([res1, res2]).then(rs =>
rs.find(r => r !== res),
const otherRes = await Promise.all([res1, res2]).then(rs =>
rs.find(r => r !== res),
);
expect(otherRes?.status).toBe(202);
// Reading subscriber 2 should now return the second event only
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { n: 2 } }],
});
});
it('should not notify subscribers that have already consumed the event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
// A single event for each subscriber, that should not be sent to the other one
await helper
.publish(
'test',
{ for: 'tester-2' },
{
notifiedSubscribers: ['tester-1'],
},
)
.expect(201);
await helper
.publish(
'test',
{ for: 'tester-1' },
{
notifiedSubscribers: ['tester-2'],
},
)
.expect(201);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-1' } }],
});
// Single client for subscriber 2 gets the event
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-2' } }],
});
});
it('should return multiple events in order', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester', ['test']).expect(201);
// A sequence of events published one at a time
for (let n = 0; n < 15; ++n) {
await helper.publish('test', { n }).expect(201);
}
// Batch size it 10
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 0 } },
{ topic: 'test', payload: { n: 1 } },
{ topic: 'test', payload: { n: 2 } },
{ topic: 'test', payload: { n: 3 } },
{ topic: 'test', payload: { n: 4 } },
{ topic: 'test', payload: { n: 5 } },
{ topic: 'test', payload: { n: 6 } },
{ topic: 'test', payload: { n: 7 } },
{ topic: 'test', payload: { n: 8 } },
{ topic: 'test', payload: { n: 9 } },
],
});
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 10 } },
{ topic: 'test', payload: { n: 11 } },
{ topic: 'test', payload: { n: 12 } },
{ topic: 'test', payload: { n: 13 } },
{ topic: 'test', payload: { n: 14 } },
],
});
});
it('should skip publishing if all subscribers have already consumed the event', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.publish(
'test',
{ for: 'tester-2' },
{ notifiedSubscribers: ['tester'] },
)
.expect(204);
});
it('should time out when no events are available', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
jest.useFakeTimers({
advanceTimers: true,
});
try {
// Can't use supertest for this one because it can't handle the partially blocking response
const res = await fetch(
`http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`,
{
headers: {
authorization: mockCredentials.service.header(),
},
},
);
expect(otherRes?.status).toBe(202);
// Reading subscriber 2 should now return the second event only
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { n: 2 } }],
});
},
);
expect(res.status).toBe(202);
it.each(databases.eachSupportedId())(
'should not notify subscribers that have already consumed the event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
const reader = res.body!.getReader();
const isClosed = () =>
Promise.race([
reader
.read()
.then(() => reader.closed)
.then(() => true),
new Promise<boolean>(r => setTimeout(r, 100, false)),
]);
// 2 subscribers
await helper.subscribe('tester-1', ['test']).expect(201);
await helper.subscribe('tester-2', ['test']).expect(201);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(true);
} finally {
jest.useRealTimers();
}
});
// A single event for each subscriber, that should not be sent to the other one
await helper
.publish(
'test',
{ for: 'tester-2' },
{
notifiedSubscribers: ['tester-1'],
},
)
.expect(201);
await helper
.publish(
'test',
{ for: 'tester-1' },
{
notifiedSubscribers: ['tester-2'],
},
)
.expect(201);
it('should refuse listen without a subscription', async () => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory()],
});
const helper = new ReqHelper(backend);
// Single client for subscriber 1 gets the event
await helper.readEvents('tester-1').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-1' } }],
});
// Single client for subscriber 2 gets the event
await helper.readEvents('tester-2').expect(200, {
events: [{ topic: 'test', payload: { for: 'tester-2' } }],
});
},
);
it.each(databases.eachSupportedId())(
'should return multiple events in order, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
// 2 subscribers
await helper.subscribe('tester', ['test']).expect(201);
// A sequence of events published one at a time
for (let n = 0; n < 15; ++n) {
await helper.publish('test', { n }).expect(201);
}
// Batch size it 10
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 0 } },
{ topic: 'test', payload: { n: 1 } },
{ topic: 'test', payload: { n: 2 } },
{ topic: 'test', payload: { n: 3 } },
{ topic: 'test', payload: { n: 4 } },
{ topic: 'test', payload: { n: 5 } },
{ topic: 'test', payload: { n: 6 } },
{ topic: 'test', payload: { n: 7 } },
{ topic: 'test', payload: { n: 8 } },
{ topic: 'test', payload: { n: 9 } },
],
});
await helper.readEvents('tester').expect(200, {
events: [
{ topic: 'test', payload: { n: 10 } },
{ topic: 'test', payload: { n: 11 } },
{ topic: 'test', payload: { n: 12 } },
{ topic: 'test', payload: { n: 13 } },
{ topic: 'test', payload: { n: 14 } },
],
});
},
);
it.each(databases.eachSupportedId())(
'should skip publishing if all subscribers have already consumed the event, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
await helper
.publish(
'test',
{ for: 'tester-2' },
{ notifiedSubscribers: ['tester'] },
)
.expect(204);
},
);
it.each(databases.eachSupportedId())(
'should time out when no events are available, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.subscribe('tester', ['test']).expect(201);
jest.useFakeTimers({
advanceTimers: true,
});
try {
// Can't use supertest for this one because it can't handle the partially blocking response
const res = await fetch(
`http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`,
{
headers: {
authorization: mockCredentials.service.header(),
},
},
);
expect(res.status).toBe(202);
const reader = res.body!.getReader();
const isClosed = () =>
Promise.race([
reader
.read()
.then(() => reader.closed)
.then(() => true),
new Promise<boolean>(r => setTimeout(r, 100, false)),
]);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(false);
await jest.advanceTimersByTimeAsync(30000);
await expect(isClosed()).resolves.toBe(true);
} finally {
jest.useRealTimers();
}
},
);
it.each(databases.eachSupportedId())(
'should refuse listen without a subscription, %p',
async databaseId => {
backend = await startTestBackend({
features: [eventsPlugin, await mockKnexFactory(databaseId)],
});
const helper = new ReqHelper(backend);
await helper.readEvents('nonexistent').expect(404);
},
);
});
});
await helper.readEvents('nonexistent').expect(404);
});
},
);
plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts
+15 -30
View File
@@ -29,13 +29,10 @@ const databases = TestDatabases.create({
ids: ['POSTGRES_14', 'POSTGRES_18'],
});
const maybeDescribe =
databases.eachSupportedId().length > 0 ? describe : describe.skip;
maybeDescribe('DatabaseEventBusStore', () => {
it.each(databases.eachSupportedId())(
'should clean up old events, %p',
async databaseId => {
describe.each(databases.eachSupportedId())(
'DatabaseEventBusStore, %p',
databaseId => {
it('should clean up old events', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({ logger, db });
@@ -79,12 +76,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events3 } = await store.readSubscription('tester-3');
expect(events3.length).toBe(5);
},
);
});
it.each(databases.eachSupportedId())(
'should always clean up events outside the max age window, %p',
async databaseId => {
it('should always clean up events outside the max age window', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -132,12 +126,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events3 } = await store.readSubscription('tester-3');
expect(events3.length).toBe(0);
},
);
});
it.each(databases.eachSupportedId())(
'should not clean up events within the min age window, %p',
async databaseId => {
it('should not clean up events within the min age window', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -162,12 +153,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
const { events: events1 } = await store.readSubscription('tester-1');
expect(events1.length).toBe(10);
},
);
});
it.each(databases.eachSupportedId())(
'should clean up a large number of events, %p',
async databaseId => {
it('should clean up a large number of events', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -197,12 +185,9 @@ maybeDescribe('DatabaseEventBusStore', () => {
await expect(db('event_bus_events').count()).resolves.toEqual([
{ count: '5' },
]);
},
);
});
it.each(databases.eachSupportedId())(
'should perform well when looking up events by topic, %p',
async databaseId => {
it('should perform well when looking up events by topic', async () => {
const db = await databases.init(databaseId);
const store = await DatabaseEventBusStore.forTest({
logger,
@@ -245,6 +230,6 @@ maybeDescribe('DatabaseEventBusStore', () => {
]);
expect(duration).toBeLessThan(20);
},
);
});
});
},
);
plugins/notifications-backend/src/tests/migrations.test.ts
+42 -45
View File
@@ -41,59 +41,56 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create();
const databases = TestDatabases.create();
it.each(databases.eachSupportedId())(
'20221109192547_search_add_original_value_column.js, %p',
async databaseId => {
const knex = await databases.init(databaseId);
describe.each(databases.eachSupportedId())('migrations, %p', databaseId => {
it('20221109192547_search_add_original_value_column.js', async () => {
const knex = await databases.init(databaseId);
await migrateUntilBefore(knex, '20250317_addTopic.js');
await migrateUntilBefore(knex, '20250317_addTopic.js');
await knex
.insert({
await knex
.insert({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
})
.into('user_settings');
await migrateUpOnce(knex);
let rows = await knex('user_settings');
let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
})
.into('user_settings');
settings_key_hash:
'73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15',
topic: null,
}),
]),
);
await migrateUpOnce(knex);
await migrateDownOnce(knex);
let rows = await knex('user_settings');
let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
rows = await knex('user_settings');
normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
settings_key_hash:
'73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15',
topic: null,
}),
]),
);
await migrateDownOnce(knex);
rows = await knex('user_settings');
normalized = rows.map(r => ({ ...r, enabled: !!r.enabled }));
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
}),
]),
);
},
);
expect(normalized).toEqual(
expect.arrayContaining([
expect.objectContaining({
user: 'user1',
channel: 'channel1',
origin: 'origin1',
enabled: true,
}),
]),
);
});
});
plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts
+598 -649
View File
File diff suppressed because it is too large Load Diff
plugins/search-backend-module-pg/src/database/util.test.ts
+33 -41
View File
@@ -18,45 +18,37 @@ import { queryPostgresMajorVersion } from './util';
jest.setTimeout(60_000);
describe('util', () => {
describe('unsupported', () => {
const databases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8'],
});
it.each(databases.eachSupportedId())(
'should fail on get postgres major version, %p',
async databaseId => {
const knex = await databases.init(databaseId);
await expect(
async () => await queryPostgresMajorVersion(knex),
).rejects.toThrow();
},
);
});
describe('supported', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14'],
});
if (databases.eachSupportedId().length < 1) {
// Only execute tests if at least on database engine is available, e.g. if
// not in CI=1. it.each doesn't support an empty array.
return;
}
it.each(databases.eachSupportedId())(
'should get postgres major version, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const expectedVersion = +databaseId.slice(9);
await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
expectedVersion,
);
},
);
});
const unsupportedDatabases = TestDatabases.create({
ids: ['SQLITE_3', 'MYSQL_8'],
});
describe.each(unsupportedDatabases.eachSupportedId())(
'util unsupported, %p',
databaseId => {
it('should fail on get postgres major version', async () => {
const knex = await unsupportedDatabases.init(databaseId);
await expect(
async () => await queryPostgresMajorVersion(knex),
).rejects.toThrow();
});
},
);
const supportedDatabases = TestDatabases.create({
ids: ['POSTGRES_18', 'POSTGRES_14'],
});
describe.each(supportedDatabases.eachSupportedId())(
'util supported, %p',
databaseId => {
it('should get postgres major version', async () => {
const knex = await supportedDatabases.init(databaseId);
const expectedVersion = +databaseId.slice(9);
await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
expectedVersion,
);
});
},
);