From c862b3f36fb8bf3ec36c2a459a19bd5ef7ed95c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 5 Mar 2021 19:33:06 +0100 Subject: [PATCH] Introduce paging in the catalog /entities endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/twelve-files-invite.md | 32 +++++ .../catalog/DatabaseEntitiesCatalog.test.ts | 109 ++++++++++------ .../src/catalog/DatabaseEntitiesCatalog.ts | 70 ++++++---- plugins/catalog-backend/src/catalog/types.ts | 26 +++- .../src/database/CommonDatabase.test.ts | 38 +++--- .../src/database/CommonDatabase.ts | 83 +++++++++++- plugins/catalog-backend/src/database/index.ts | 1 + plugins/catalog-backend/src/database/types.ts | 33 ++++- .../ingestion/HigherOrderOperations.test.ts | 5 +- .../src/service/CatalogBuilder.test.ts | 9 +- .../src/service/EntityFilters.test.ts | 113 ----------------- .../src/service/EntityFilters.ts | 120 ------------------ .../src/service/request/basicEntityFilter.ts | 39 ++++++ .../src/service/request/common.ts | 79 ++++++++++++ .../src/service/request/index.ts | 20 +++ .../request/parseEntityFilterParams.test.ts | 99 +++++++++++++++ .../request/parseEntityFilterParams.ts | 86 +++++++++++++ .../parseEntityPaginationParams.test.ts | 23 ++++ .../request/parseEntityPaginationParams.ts | 40 ++++++ .../parseEntityTransformParams.test.ts} | 31 ++--- .../parseEntityTransformParams.ts} | 34 ++--- .../src/service/router.test.ts | 87 ++++++++----- plugins/catalog-backend/src/service/router.ts | 48 ++++--- 23 files changed, 805 insertions(+), 420 deletions(-) create mode 100644 .changeset/twelve-files-invite.md delete mode 100644 plugins/catalog-backend/src/service/EntityFilters.test.ts delete mode 100644 plugins/catalog-backend/src/service/EntityFilters.ts create mode 100644 plugins/catalog-backend/src/service/request/basicEntityFilter.ts create mode 100644 plugins/catalog-backend/src/service/request/common.ts create mode 100644 plugins/catalog-backend/src/service/request/index.ts create mode 100644 plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts create mode 100644 plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts create mode 100644 plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts create mode 100644 plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts rename plugins/catalog-backend/src/service/{filterQuery.test.ts => request/parseEntityTransformParams.test.ts} (55%) rename plugins/catalog-backend/src/service/{filterQuery.ts => request/parseEntityTransformParams.ts} (66%) diff --git a/.changeset/twelve-files-invite.md b/.changeset/twelve-files-invite.md new file mode 100644 index 0000000000..8009e92110 --- /dev/null +++ b/.changeset/twelve-files-invite.md @@ -0,0 +1,32 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Introduce pagination in the /entities catalog endpoint. + +Pagination is requested using query parameters. Currently supported parameters, all optional, are: + +- `limit` - an integer number of entities to return, at most +- `offset` - an integer number of entities to skip over at the start +- `after` - an opaque string cursor as returned by a previous paginated request + +Example request: + +`GET /entities?limit=100` + +Example response: + +``` +200 OK +Content-Type: application/json; charset=utf-8 +Link: ; rel="next" + + +[{"metadata":{... +``` + +Note the Link header. It contains the URL (path and query part, relative to the catalog root) to use for requesting the next page. +It uses the `after` cursor to point out the end of the previous page. If the Link header is not present, there is no more data to read. + +The current implementation is naive and encodes offset/limit in the cursor implementation, so it is not robust in the face of overlapping +changes to the catalog. This can be improved separately in the future without having to change the calling patterns. diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts index 5059d741f4..2d37e5b334 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model'; import { Database, DatabaseManager, Transaction } from '../database'; -import { EntityFilters } from '../service/EntityFilters'; +import { basicEntityFilter } from '../service/request'; import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; import { EntityUpsertRequest } from './types'; @@ -63,7 +63,10 @@ describe('DatabaseEntitiesCatalog', () => { }, }; - db.entities.mockResolvedValue([]); + db.entities.mockResolvedValue({ + entities: [], + pageInfo: { hasNext: false }, + }); db.addEntities.mockResolvedValue([ { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, ]); @@ -74,12 +77,13 @@ describe('DatabaseEntitiesCatalog', () => { ]); expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith( - expect.anything(), - EntityFilters.ofFilterString( - 'kind=b,metadata.namespace=d,metadata.name=c', - ), - ); + expect(db.entities).toHaveBeenCalledWith(expect.anything(), { + filter: basicEntityFilter({ + kind: 'b', + 'metadata.namespace': 'd', + 'metadata.name': 'c', + }), + }); expect(db.addEntities).toHaveBeenCalledTimes(1); expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [ { entity: expect.anything(), relations: [] }, @@ -96,7 +100,10 @@ describe('DatabaseEntitiesCatalog', () => { namespace: 'd', }, }; - db.entities.mockResolvedValue([]); + db.entities.mockResolvedValue({ + entities: [], + pageInfo: { hasNext: false }, + }); db.addEntities.mockResolvedValue([ { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, ]); @@ -108,12 +115,13 @@ describe('DatabaseEntitiesCatalog', () => { ); expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith( - expect.anything(), - EntityFilters.ofFilterString( - 'kind=b,metadata.namespace=d,metadata.name=c', - ), - ); + expect(db.entities).toHaveBeenCalledWith(expect.anything(), { + filter: basicEntityFilter({ + kind: 'b', + 'metadata.namespace': 'd', + 'metadata.name': 'c', + }), + }); expect(db.addEntities).toHaveBeenCalledTimes(1); expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [ { entity: expect.anything(), relations: [] }, @@ -147,7 +155,10 @@ describe('DatabaseEntitiesCatalog', () => { }, }, }; - db.entities.mockResolvedValue([{ entity: dbEntity }]); + db.entities.mockResolvedValue({ + entities: [{ entity: dbEntity }], + pageInfo: { hasNext: false }, + }); db.addEntities.mockResolvedValue([ { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } }, ]); @@ -198,7 +209,10 @@ describe('DatabaseEntitiesCatalog', () => { }, }; - db.entities.mockResolvedValue([existing]); + db.entities.mockResolvedValue({ + entities: [existing], + pageInfo: { hasNext: false }, + }); db.entityByUid.mockResolvedValue(existing); db.updateEntity.mockResolvedValue({ entity }); @@ -208,12 +222,13 @@ describe('DatabaseEntitiesCatalog', () => { ]); expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith( - expect.anything(), - EntityFilters.ofFilterString( - 'kind=b,metadata.namespace=d,metadata.name=c', - ), - ); + expect(db.entities).toHaveBeenCalledWith(expect.anything(), { + filter: basicEntityFilter({ + kind: 'b', + 'metadata.namespace': 'd', + 'metadata.name': 'c', + }), + }); expect(db.entityByName).not.toHaveBeenCalled(); expect(db.entityByUid).toHaveBeenCalledTimes(1); expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u'); @@ -272,7 +287,10 @@ describe('DatabaseEntitiesCatalog', () => { }, }; - db.entities.mockResolvedValue([existing]); + db.entities.mockResolvedValue({ + entities: [existing], + pageInfo: { hasNext: false }, + }); db.entityByName.mockResolvedValue(existing); db.updateEntity.mockResolvedValue(existing); @@ -282,12 +300,13 @@ describe('DatabaseEntitiesCatalog', () => { ]); expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith( - expect.anything(), - EntityFilters.ofFilterString( - 'kind=b,metadata.namespace=d,metadata.name=c', - ), - ); + expect(db.entities).toHaveBeenCalledWith(expect.anything(), { + filter: basicEntityFilter({ + kind: 'b', + 'metadata.namespace': 'd', + 'metadata.name': 'c', + }), + }); expect(db.entityByName).toHaveBeenCalledTimes(1); expect(db.entityByName).toHaveBeenCalledWith(transaction, { kind: 'b', @@ -334,7 +353,10 @@ describe('DatabaseEntitiesCatalog', () => { }, }; - db.entities.mockResolvedValue([{ entity }]); + db.entities.mockResolvedValue({ + entities: [{ entity }], + pageInfo: { hasNext: false }, + }); db.entityByUid.mockResolvedValue({ entity }); db.updateEntity.mockResolvedValue({ entity }); @@ -344,12 +366,13 @@ describe('DatabaseEntitiesCatalog', () => { ]); expect(db.entities).toHaveBeenCalledTimes(1); - expect(db.entities).toHaveBeenCalledWith( - expect.anything(), - EntityFilters.ofFilterString( - 'kind=b,metadata.namespace=d,metadata.name=c', - ), - ); + expect(db.entities).toHaveBeenCalledWith(expect.anything(), { + filter: basicEntityFilter({ + kind: 'b', + 'metadata.namespace': 'd', + 'metadata.name': 'c', + }), + }); expect(db.entityByName).not.toHaveBeenCalled(); expect(db.entityByUid).not.toHaveBeenCalled(); expect(db.updateEntity).not.toHaveBeenCalled(); @@ -377,7 +400,7 @@ describe('DatabaseEntitiesCatalog', () => { await catalog.batchAddOrUpdateEntities(entities); const afterFirst = await catalog.entities(); - expect(afterFirst.length).toBe(300); + expect(afterFirst.entities.length).toBe(300); entities[40].entity.metadata.op = 'changed'; entities.push({ @@ -391,9 +414,13 @@ describe('DatabaseEntitiesCatalog', () => { await catalog.batchAddOrUpdateEntities(entities); const afterSecond = await catalog.entities(); - expect(afterSecond.length).toBe(301); - expect(afterSecond.find(e => e.metadata.op === 'changed')).toBeDefined(); - expect(afterSecond.find(e => e.metadata.op === 'added')).toBeDefined(); + expect(afterSecond.entities.length).toBe(301); + expect( + afterSecond.entities.find(e => e.metadata.op === 'changed'), + ).toBeDefined(); + expect( + afterSecond.entities.find(e => e.metadata.op === 'added'), + ).toBeDefined(); }, 10000); }); }); diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 7cc88b9f18..c53bf751a0 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { ConflictError, NotFoundError } from '@backstage/errors'; import { Entity, entityHasChanges, @@ -23,19 +22,18 @@ import { LOCATION_ANNOTATION, serializeEntityRef, } from '@backstage/catalog-model'; +import { ConflictError, NotFoundError } from '@backstage/errors'; import { chunk, groupBy } from 'lodash'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; -import type { - Database, - DbEntityResponse, - EntityFilter, - Transaction, -} from '../database'; -import { EntityFilters } from '../service/EntityFilters'; +import type { Database, DbEntityResponse, Transaction } from '../database'; +import { DbEntitiesRequest } from '../database/types'; +import { basicEntityFilter } from '../service/request'; import { durationText } from '../util/timing'; import type { EntitiesCatalog, + EntitiesRequest, + EntitiesResponse, EntityUpsertRequest, EntityUpsertResponse, } from './types'; @@ -65,11 +63,24 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { private readonly logger: Logger, ) {} - async entities(filter?: EntityFilter): Promise { - const items = await this.database.transaction(tx => - this.database.entities(tx, filter), + async entities(request?: EntitiesRequest): Promise { + const dbRequest: DbEntitiesRequest = { + filter: request?.filter, + pagination: request?.pagination, + }; + + const dbResponse = await this.database.transaction(tx => + this.database.entities(tx, dbRequest), ); - return items.map(i => i.entity); + + const entities = dbResponse.entities.map(e => + request?.fields ? request.fields(e.entity) : e.entity, + ); + + return { + entities, + pageInfo: dbResponse.pageInfo, + }; } async removeEntityByUid(uid: string): Promise { @@ -78,16 +89,20 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { if (!entityResponse) { throw new NotFoundError(`Entity with ID ${uid} was not found`); } + const location = entityResponse.entity.metadata.annotations?.[LOCATION_ANNOTATION]; + const colocatedEntities = location - ? await this.database.entities( - tx, - EntityFilters.ofMatchers({ - [`metadata.annotations.${LOCATION_ANNOTATION}`]: location, - }), - ) + ? ( + await this.database.entities(tx, { + filter: basicEntityFilter({ + [`metadata.annotations.${LOCATION_ANNOTATION}`]: location, + }), + }) + ).entities : [entityResponse]; + for (const dbResponse of colocatedEntities) { await this.database.removeEntityByUid( tx, @@ -98,6 +113,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { if (entityResponse.locationId) { await this.database.removeLocation(tx, entityResponse?.locationId!); } + return undefined; }); } @@ -206,13 +222,12 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { } if (options?.outputEntities && responses.length > 0) { - const writtenEntities = await this.database.entities( - tx, - EntityFilters.ofMatchers({ + const writtenEntities = await this.database.entities(tx, { + filter: basicEntityFilter({ 'metadata.uid': responses.map(e => e.entityId), }), - ); - responses = writtenEntities.map(e => ({ + }); + responses = writtenEntities.entities.map(e => ({ entityId: e.entity.metadata.uid!, entity: e.entity, })); @@ -247,17 +262,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { // Here we make use of the fact that all of the entities share kind and // namespace within a batch const names = requests.map(({ entity }) => entity.metadata.name); - const oldEntities = await this.database.entities( - tx, - EntityFilters.ofMatchers({ + const oldEntitiesResponse = await this.database.entities(tx, { + filter: basicEntityFilter({ kind: kind, 'metadata.namespace': namespace, 'metadata.name': names, }), - ); + }); const oldEntitiesByName = new Map( - oldEntities.map(e => [e.entity.metadata.name, e.entity]), + oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]), ); const toAdd: EntityUpsertRequest[] = []; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index b012ee3606..a474fd4f66 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,12 +15,32 @@ */ import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model'; -import type { EntityFilter } from '../database'; +import { EntityFilter, EntityPagination } from '../database/types'; // // Entities // +export type PageInfo = + | { + hasNext: false; + } + | { + hasNext: true; + endCursor: string; + }; + +export type EntitiesRequest = { + filter?: EntityFilter; + fields?: (entity: Entity) => Entity; + pagination?: EntityPagination; +}; + +export type EntitiesResponse = { + entities: Entity[]; + pageInfo: PageInfo; +}; + export type EntityUpsertRequest = { entity: Entity; relations: EntityRelationSpec[]; @@ -35,9 +55,9 @@ export type EntitiesCatalog = { /** * Fetch entities. * - * @param filter A filter to apply when reading + * @param request Request options */ - entities(filter?: EntityFilter): Promise; + entities(request?: EntitiesRequest): Promise; /** * Removes a single entity. diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index 0e26384358..27da003791 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import { ConflictError } from '@backstage/errors'; import { Entity, Location, parseEntityRef } from '@backstage/catalog-model'; -import { EntityFilters } from '../service/EntityFilters'; +import { ConflictError } from '@backstage/errors'; +import { basicEntityFilter } from '../service/request'; import { DatabaseManager } from './DatabaseManager'; import type { DbEntityRequest, @@ -385,8 +385,8 @@ describe('CommonDatabase', () => { ]); }); const result = await db.transaction(async tx => db.entities(tx)); - expect(result.length).toEqual(2); - expect(result).toEqual( + expect(result.entities.length).toEqual(2); + expect(result.entities).toEqual( expect.arrayContaining([ { locationId: undefined, @@ -424,11 +424,13 @@ describe('CommonDatabase', () => { ); }); - await expect( - db.transaction(async tx => - db.entities(tx, EntityFilters.ofFilterString('kind=k2,spec.c=some')), - ), - ).resolves.toEqual([ + const response = await db.transaction(async tx => + db.entities(tx, { + filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }), + }), + ); + + expect(response.entities).toEqual([ { locationId: undefined, entity: expect.objectContaining({ kind: 'k2' }), @@ -466,14 +468,13 @@ describe('CommonDatabase', () => { }); const rows = await db.transaction(async tx => - db.entities( - tx, - EntityFilters.ofFilterString('ApiVersioN=A,spEc.C=some'), - ), + db.entities(tx, { + filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }), + }), ); - expect(rows.length).toEqual(3); - expect(rows).toEqual( + expect(rows.entities.length).toEqual(3); + expect(rows.entities).toEqual( expect.arrayContaining([ { locationId: undefined, @@ -531,7 +532,8 @@ describe('CommonDatabase', () => { { target: mockRelations[0].target, type: 'child' }, ]); - const [returnedEntity3] = await db.transaction(tx => db.entities(tx)); + const { entities } = await db.transaction(tx => db.entities(tx)); + const [returnedEntity3] = entities; expect(returnedEntity3?.entity.relations).toEqual([ { target: mockRelations[0].target, type: 'child' }, ]); @@ -615,7 +617,7 @@ describe('CommonDatabase', () => { const res = await db.transaction(tx => db.entities(tx)); expect( - res.map(r => ({ + res.entities.map(r => ({ name: r.entity.metadata.name, relations: r.entity.relations, })), @@ -660,7 +662,7 @@ describe('CommonDatabase', () => { const res2 = await db.transaction(tx => db.entities(tx)); expect( - res2.map(r => ({ + res2.entities.map(r => ({ name: r.entity.metadata.name, relations: r.entity.relations, })), diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index 890ce48e0c..6b1beae538 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -35,13 +35,16 @@ import { DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, DbEntitiesRelationsRow, + DbEntitiesRequest, + DbEntitiesResponse, DbEntitiesRow, DbEntitiesSearchRow, DbEntityRequest, DbEntityResponse, DbLocationsRow, DbLocationsRowWithStatus, - EntityFilter, + DbPageInfo, + EntityPagination, Transaction, } from './types'; @@ -207,13 +210,13 @@ export class CommonDatabase implements Database { async entities( txOpaque: Transaction, - filter?: EntityFilter, - ): Promise { + request?: DbEntitiesRequest, + ): Promise { const tx = txOpaque as Knex.Transaction; let entitiesQuery = tx('entities'); - for (const singleFilter of filter?.anyOf ?? []) { + for (const singleFilter of request?.filter?.anyOf ?? []) { entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() { for (const { key, matchValueIn } of singleFilter.allOf) { // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to @@ -235,17 +238,43 @@ export class CommonDatabase implements Database { } } }); - this.andWhere('id', 'in', matchQuery); } }); } - const rows = await entitiesQuery + entitiesQuery = entitiesQuery .select('entities.*') .orderBy('full_name', 'asc'); - return this.toEntityResponses(tx, rows); + const { limit, offset } = parsePagination(request?.pagination); + if (limit !== undefined) { + entitiesQuery = entitiesQuery.limit(limit + 1); + } + if (offset !== undefined) { + entitiesQuery = entitiesQuery.offset(offset); + } + + let rows = await entitiesQuery; + + let pageInfo: DbPageInfo; + if (limit === undefined || rows.length <= limit) { + pageInfo = { hasNext: false }; + } else { + rows = rows.slice(0, -1); + pageInfo = { + hasNext: true, + endCursor: stringifyPagination({ + limit, + offset: (offset ?? 0) + limit, + }), + }; + } + + return { + entities: await this.toEntityResponses(tx, rows), + pageInfo, + }; } async entityByName( @@ -519,6 +548,46 @@ export class CommonDatabase implements Database { } } +function parsePagination( + input?: EntityPagination, +): { limit?: number; offset?: number } { + if (!input) { + return {}; + } + + let { limit, offset } = input; + + if (input.after !== undefined) { + let cursor; + try { + const json = Buffer.from(input.after, 'base64').toString('utf8'); + cursor = JSON.parse(json); + } catch { + throw new InputError('Malformed after cursor'); + } + if (cursor.limit !== undefined) { + if (!Number.isInteger(cursor.limit)) { + throw new InputError('Malformed after cursor'); + } + limit = cursor.limit; + } + if (cursor.offset !== undefined) { + if (!Number.isInteger(cursor.offset)) { + throw new InputError('Malformed after cursor'); + } + offset = cursor.offset; + } + } + + return { limit, offset }; +} + +function stringifyPagination(input: { limit: number; offset: number }) { + const json = JSON.stringify({ limit: input.limit, offset: input.offset }); + const base64 = Buffer.from(json, 'utf8').toString('base64'); + return base64; +} + function deduplicateRelations( rows: DbEntitiesRelationsRow[], ): DbEntitiesRelationsRow[] { diff --git a/plugins/catalog-backend/src/database/index.ts b/plugins/catalog-backend/src/database/index.ts index c2cc07788a..edc8c56ac2 100644 --- a/plugins/catalog-backend/src/database/index.ts +++ b/plugins/catalog-backend/src/database/index.ts @@ -22,5 +22,6 @@ export type { DbEntityResponse, EntitiesSearchFilter, EntityFilter, + EntityPagination, Transaction, } from './types'; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ee2d4222c3..234d3c0d8d 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -36,6 +36,25 @@ export type DbEntityRequest = { relations: EntityRelationSpec[]; }; +export type DbEntitiesRequest = { + filter?: EntityFilter; + pagination?: EntityPagination; +}; + +export type DbEntitiesResponse = { + entities: DbEntityResponse[]; + pageInfo: DbPageInfo; +}; + +export type DbPageInfo = + | { + hasNext: false; + } + | { + hasNext: true; + endCursor: string; + }; + export type DbEntityResponse = { locationId?: string; entity: Entity; @@ -111,6 +130,15 @@ export type EntityFilter = { anyOf: { allOf: EntitiesSearchFilter[] }[]; }; +/** + * A pagination rule for entities. + */ +export type EntityPagination = { + limit?: number; + offset?: number; + after?: string; +}; + /** * An abstraction for transactions of the underlying database technology. */ @@ -170,7 +198,10 @@ export type Database = { matchingGeneration?: number, ): Promise; - entities(tx: Transaction, filter?: EntityFilter): Promise; + entities( + tx: Transaction, + request?: DbEntitiesRequest, + ): Promise; entityByName( tx: Transaction, diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts index b9078df93f..4f4980a33b 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts @@ -360,7 +360,10 @@ describe('HigherOrderOperations', () => { entities: [{ entity: desc, location, relations: [] }], errors: [], }); - entitiesCatalog.entities.mockResolvedValue([]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [], + pageInfo: { hasNext: false }, + }); entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]); await expect( diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index cdc01ba60e..cf437f0a21 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -61,7 +61,10 @@ describe('CatalogBuilder', () => { it('works with no changes', async () => { const builder = new CatalogBuilder(env); const built = await builder.build(); - await expect(built.entitiesCatalog.entities()).resolves.toEqual([]); + await expect(built.entitiesCatalog.entities()).resolves.toEqual({ + entities: [], + pageInfo: { hasNext: false }, + }); await expect(built.locationsCatalog.locations()).resolves.toEqual([ expect.objectContaining({ data: expect.objectContaining({ type: 'bootstrap' }), @@ -166,7 +169,7 @@ describe('CatalogBuilder', () => { type: 'github', target: 'https://github.com/a/b/x.yaml', }); - const entities = await entitiesCatalog.entities(); + const { entities } = await entitiesCatalog.entities(); expect(entities).toEqual([ expect.objectContaining({ @@ -200,7 +203,7 @@ describe('CatalogBuilder', () => { type: 'x', target: 'y', }); - const entities = await entitiesCatalog.entities(); + const { entities } = await entitiesCatalog.entities(); expect.assertions(3); expect(entities).toEqual([ diff --git a/plugins/catalog-backend/src/service/EntityFilters.test.ts b/plugins/catalog-backend/src/service/EntityFilters.test.ts deleted file mode 100644 index bfc20c625a..0000000000 --- a/plugins/catalog-backend/src/service/EntityFilters.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { EntityFilters } from './EntityFilters'; - -describe('EntityFilters', () => { - describe('ofQuery', () => { - it('translates empty query to empty list', () => { - const result = EntityFilters.ofQuery({}); - expect(result).toEqual(undefined); - }); - - it('supports single-string format', () => { - const result = EntityFilters.ofQuery({ filter: 'a=1' })!; - expect(result).toEqual({ - anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }], - }); - }); - - it('supports array-of-strings format', () => { - const result = EntityFilters.ofQuery({ filter: ['a=1', 'b=2'] }); - expect(result).toEqual({ - anyOf: [ - { allOf: [{ key: 'a', matchValueIn: ['1'] }] }, - { allOf: [{ key: 'b', matchValueIn: ['2'] }] }, - ], - }); - }); - - it('throws for non-strings', () => { - expect(() => EntityFilters.ofQuery({ filter: [3] })).toThrow(/string/); - }); - }); - - describe('ofFilterString', () => { - it('runs the happy path', () => { - const result = EntityFilters.ofFilterString('a=1,b=2'); - expect(result).toEqual({ - anyOf: [ - { - allOf: [ - { key: 'a', matchValueIn: ['1'] }, - { key: 'b', matchValueIn: ['2'] }, - ], - }, - ], - }); - }); - - it('ignores empty', () => { - const result = EntityFilters.ofFilterString('a=1,,b=2,'); - expect(result).toEqual({ - anyOf: [ - { - allOf: [ - { key: 'a', matchValueIn: ['1'] }, - { key: 'b', matchValueIn: ['2'] }, - ], - }, - ], - }); - }); - - it('trims', () => { - const result = EntityFilters.ofFilterString(' a = 1 ,, b=2 ,'); - expect(result).toEqual({ - anyOf: [ - { - allOf: [ - { key: 'a', matchValueIn: ['1'] }, - { key: 'b', matchValueIn: ['2'] }, - ], - }, - ], - }); - }); - - it('merges multiple of the same key', () => { - const result = EntityFilters.ofFilterString('a=1,a=2,b=3'); - expect(result).toEqual({ - anyOf: [ - { - allOf: [ - { key: 'a', matchValueIn: ['1', '2'] }, - { key: 'b', matchValueIn: ['3'] }, - ], - }, - ], - }); - }); - - it('throws on missing equal sign', () => { - expect(() => EntityFilters.ofFilterString('a,b=2')).toThrow(); - }); - - it('throws on misplaced equal sign', () => { - expect(() => EntityFilters.ofFilterString('a=,b=2')).toThrow(); - }); - }); -}); diff --git a/plugins/catalog-backend/src/service/EntityFilters.ts b/plugins/catalog-backend/src/service/EntityFilters.ts deleted file mode 100644 index 83c0bfea6c..0000000000 --- a/plugins/catalog-backend/src/service/EntityFilters.ts +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { InputError } from '@backstage/errors'; -import { EntitiesSearchFilter, EntityFilter } from '../database'; - -/** - * A builder that assists in creating entity filter instances. - */ -export class EntityFilters { - // Builds a filter from the value of a filter=a=1,b=2 type filter string - static ofFilterString(filterString: string): EntityFilter | undefined { - const builder = new EntityFilters(); - builder.addFilterString(filterString); - return builder.build(); - } - - // Builds a filter from an entire query params structure - static ofQuery(query: Record): EntityFilter | undefined { - const filterStrings = [query.filter || []].flat(); - - if (filterStrings.some(f => typeof f !== 'string')) { - throw new InputError( - 'Only string type filter query parameters are supported', - ); - } - - const builder = new EntityFilters(); - for (const filterString of filterStrings) { - builder.addFilterString(filterString); - } - - return builder.build(); - } - - // Builds a filter from a { key: value } matcher object - static ofMatchers( - matchers: Record, - ): EntityFilter | undefined { - const builder = new EntityFilters(); - builder.addMatchers(matchers); - return builder.build(); - } - - private filters: EntitiesSearchFilter[][]; - - constructor() { - this.filters = []; - } - - addFilterString(filterString: string): EntityFilters { - const filtersByKey: Record = {}; - - const addFilter = (key: string, value: string) => { - const f = - key in filtersByKey - ? filtersByKey[key] - : (filtersByKey[key] = { key, matchValueIn: [] }); - f.matchValueIn!.push(value); - }; - - const statements = filterString - .split(',') - .map(s => s.trim()) - .filter(Boolean); - - for (const statement of statements) { - const equalsIndex = statement.indexOf('='); - if (equalsIndex < 1) { - throw new InputError('Malformed filter query'); - } else { - const key = statement.substr(0, equalsIndex).trim(); - const value = statement.substr(equalsIndex + 1).trim(); - if (!key || !value) { - throw new InputError('Malformed filter query'); - } - addFilter(key, value); - } - } - - this.filters.push(Object.values(filtersByKey)); - - return this; - } - - addMatchers(matchers: Record): EntityFilters { - const filters: EntitiesSearchFilter[] = []; - - for (const [key, value] of Object.entries(matchers)) { - filters.push({ key, matchValueIn: [value].flat() }); - } - - if (filters.length) { - this.filters.push(filters); - } - - return this; - } - - build(): EntityFilter | undefined { - if (!this.filters.length) { - return undefined; - } - - return { anyOf: this.filters.map(f => ({ allOf: f })) }; - } -} diff --git a/plugins/catalog-backend/src/service/request/basicEntityFilter.ts b/plugins/catalog-backend/src/service/request/basicEntityFilter.ts new file mode 100644 index 0000000000..06f2c013b3 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/basicEntityFilter.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntitiesSearchFilter, EntityFilter } from '../../database'; + +/** + * Forms a full EntityFilter based on a single key-value(s) object. + */ +export function basicEntityFilter( + items: Record, +): EntityFilter { + const filtersByKey: Record = {}; + + for (const [key, value] of Object.entries(items)) { + const values = [value].flat(); + + const f = + key in filtersByKey + ? filtersByKey[key] + : (filtersByKey[key] = { key, matchValueIn: [] }); + + f.matchValueIn!.push(...values); + } + + return { anyOf: [{ allOf: Object.values(filtersByKey) }] }; +} diff --git a/plugins/catalog-backend/src/service/request/common.ts b/plugins/catalog-backend/src/service/request/common.ts new file mode 100644 index 0000000000..81369d032b --- /dev/null +++ b/plugins/catalog-backend/src/service/request/common.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; + +/** + * Takes a single unknown parameter and makes sure that it's a string that can + * be parsed as an integer. + */ +export function parseIntegerParam( + param: unknown, + ctx: string, +): number | undefined { + if (param === undefined) { + return undefined; + } + + if (typeof param !== 'string') { + throw new InputError(`Invalid ${ctx}, not an integer on string form`); + } + + const parsed = parseInt(param, 10); + if (!Number.isInteger(parsed) || String(parsed) !== param) { + throw new InputError(`Invalid ${ctx}, not an integer`); + } + + return parsed; +} + +/** + * Takes a single unknown parameter and makes sure that it's a string. + */ +export function parseStringParam( + param: unknown, + ctx: string, +): string | undefined { + if (param === undefined) { + return undefined; + } + + if (typeof param !== 'string') { + throw new InputError(`Invalid ${ctx}, not a string`); + } + + return param; +} + +/** + * Takes a single unknown parameter and makes sure that it's a single string or + * an array of strings, and returns as an array. + */ +export function parseStringsParam( + param: unknown, + ctx: string, +): string[] | undefined { + if (param === undefined) { + return undefined; + } + + const array = [param].flat(); + if (array.some(p => typeof p !== 'string')) { + throw new InputError(`Invalid ${ctx}, not a string`); + } + + return array as string[]; +} diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts new file mode 100644 index 0000000000..c2c51f9ab7 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { basicEntityFilter } from './basicEntityFilter'; +export { parseEntityFilterParams } from './parseEntityFilterParams'; +export { parseEntityPaginationParams } from './parseEntityPaginationParams'; +export { parseEntityTransformParams } from './parseEntityTransformParams'; diff --git a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts new file mode 100644 index 0000000000..c44edd7e9b --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + parseEntityFilterParams, + parseEntityFilterString, +} from './parseEntityFilterParams'; + +describe('parseEntityFilterParams', () => { + it('translates empty query to empty list', () => { + const result = parseEntityFilterParams({}); + expect(result).toEqual(undefined); + }); + + it('supports single-string format', () => { + const result = parseEntityFilterParams({ filter: 'a=1' })!; + expect(result).toEqual({ + anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }], + }); + }); + + it('supports array-of-strings format', () => { + const result = parseEntityFilterParams({ + filter: ['a=1', 'b=2'], + }); + expect(result).toEqual({ + anyOf: [ + { allOf: [{ key: 'a', matchValueIn: ['1'] }] }, + { allOf: [{ key: 'b', matchValueIn: ['2'] }] }, + ], + }); + }); + + it('merges values within each filter', () => { + const result = parseEntityFilterParams({ + filter: ['a=1', 'b=2,b=3,c=4'], + }); + expect(result).toEqual({ + anyOf: [ + { allOf: [{ key: 'a', matchValueIn: ['1'] }] }, + { + allOf: [ + { key: 'b', matchValueIn: ['2', '3'] }, + { key: 'c', matchValueIn: ['4'] }, + ], + }, + ], + }); + }); + + it('throws for non-strings', () => { + expect(() => parseEntityFilterParams({ filter: [3] })).toThrow(/string/); + }); +}); + +describe('parseEntityFilterString', () => { + it('works for the happy path', () => { + expect(parseEntityFilterString('')).toBeUndefined(); + expect(parseEntityFilterString('a=1,b=2,a=3')).toEqual([ + { key: 'a', matchValueIn: ['1', '3'] }, + { key: 'b', matchValueIn: ['2'] }, + ]); + }); + + it('trims values', () => { + expect(parseEntityFilterString(' a = 1 , b = 2 , a = 3 ')).toEqual([ + { key: 'a', matchValueIn: ['1', '3'] }, + { key: 'b', matchValueIn: ['2'] }, + ]); + }); + + it('rejects malformed strings', () => { + expect(() => parseEntityFilterString('x=2,a=')).toThrow( + "Invalid filter, 'a=' is not a valid statement (expected a string on the form a=b)", + ); + expect(() => parseEntityFilterString('x=2,=a')).toThrow( + "Invalid filter, '=a' is not a valid statement (expected a string on the form a=b)", + ); + expect(() => parseEntityFilterString('x=2,=')).toThrow( + "Invalid filter, '=' is not a valid statement (expected a string on the form a=b)", + ); + expect(() => parseEntityFilterString('x=2,a')).toThrow( + "Invalid filter, 'a' is not a valid statement (expected a string on the form a=b)", + ); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts new file mode 100644 index 0000000000..726eab9382 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { EntitiesSearchFilter, EntityFilter } from '../../database'; +import { parseStringsParam } from './common'; + +/** + * Parses the filtering part of a query, like + * /entities?filter=metadata.namespace=default,kind=Component + */ +export function parseEntityFilterParams( + params: Record, +): EntityFilter | undefined { + // Each filter string is on the form a=b,c=d + const filterStrings = parseStringsParam(params.filter, 'filter'); + if (!filterStrings) { + return undefined; + } + + // Outer array: "any of the inner ones" + // Inner arrays: "all of these must match" + const filters = filterStrings.map(parseEntityFilterString).filter(Boolean); + if (!filters.length) { + return undefined; + } + + return { anyOf: filters.map(f => ({ allOf: f! })) }; +} + +/** + * Parses a single filter string as seen in a filter query, for example + * metadata.namespace=default,kind=Component + */ +export function parseEntityFilterString( + filterString: string, +): EntitiesSearchFilter[] | undefined { + const statements = filterString + .split(',') + .map(s => s.trim()) + .filter(Boolean); + + if (!statements.length) { + return undefined; + } + + const filtersByKey: Record = {}; + + for (const statement of statements) { + const equalsIndex = statement.indexOf('='); + if (equalsIndex < 1) { + throw new InputError( + `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`, + ); + } + + const key = statement.substr(0, equalsIndex).trim(); + const value = statement.substr(equalsIndex + 1).trim(); + if (!key || !value) { + throw new InputError( + `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`, + ); + } + + const f = + key in filtersByKey + ? filtersByKey[key] + : (filtersByKey[key] = { key, matchValueIn: [] }); + f.matchValueIn!.push(value); + } + + return Object.values(filtersByKey); +} diff --git a/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts new file mode 100644 index 0000000000..c6f6942344 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { parseEntityPaginationParams } from './parseEntityPaginationParams'; + +describe('parseEntityPaginationParams', () => { + it('works for the happy path', () => { + expect(parseEntityPaginationParams({})).toBeUndefined(); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts new file mode 100644 index 0000000000..57e7ff59e3 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPagination } from '../../database'; +import { parseIntegerParam, parseStringParam } from './common'; + +/** + * Parses the pagination related parameters out of a query, e.g. + * /entities?offset=100&limit=10 + */ +export function parseEntityPaginationParams( + params: Record, +): EntityPagination | undefined { + const offset = parseIntegerParam(params.offset, 'offset'); + const limit = parseIntegerParam(params.limit, 'limit'); + const after = parseStringParam(params.after, 'after'); + + if (offset === undefined && limit === undefined && after === undefined) { + return undefined; + } + + return { + ...(offset ? { offset } : {}), + ...(limit ? { limit } : {}), + ...(after ? { after } : {}), + }; +} diff --git a/plugins/catalog-backend/src/service/filterQuery.test.ts b/plugins/catalog-backend/src/service/request/parseEntityTransformParams.test.ts similarity index 55% rename from plugins/catalog-backend/src/service/filterQuery.test.ts rename to plugins/catalog-backend/src/service/request/parseEntityTransformParams.test.ts index a4b3d9c671..b8b43a5098 100644 --- a/plugins/catalog-backend/src/service/filterQuery.test.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityTransformParams.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,9 @@ */ import { Entity } from '@backstage/catalog-model'; -import { translateQueryToFieldMapper } from './filterQuery'; +import { parseEntityTransformParams } from './parseEntityTransformParams'; -describe('translateQueryToFieldMapper', () => { +describe('parseEntityTransformParams', () => { const entity: Entity = { apiVersion: 'av', kind: 'k', @@ -30,37 +30,38 @@ describe('translateQueryToFieldMapper', () => { }, }; - it('passes through when no fields given', () => { - expect(translateQueryToFieldMapper({})(entity)).toBe(entity); - expect(translateQueryToFieldMapper({ fields: [] })(entity)).toBe(entity); - expect(translateQueryToFieldMapper({ fields: [''] })(entity)).toBe(entity); - expect(translateQueryToFieldMapper({ fields: [','] })(entity)).toBe(entity); + it('returns undefined when no fields given', () => { + expect(parseEntityTransformParams({})).toBeUndefined(); + expect(parseEntityTransformParams({ fields: '' })).toBeUndefined(); + expect(parseEntityTransformParams({ fields: [] })).toBeUndefined(); + expect(parseEntityTransformParams({ fields: [''] })).toBeUndefined(); + expect(parseEntityTransformParams({ fields: [','] })).toBeUndefined(); }); it('rejects attempts at array filtering', () => { expect(() => - translateQueryToFieldMapper({ fields: 'metadata.tags[0]' })(entity), - ).toThrow(/array/i); + parseEntityTransformParams({ fields: 'metadata.tags[0]' })!(entity), + ).toThrow(/invalid fields, array type fields are not supported/i); }); it('accepts both strings and arrays of strings as input', () => { - expect(translateQueryToFieldMapper({ fields: 'kind' })(entity)).toEqual({ + expect(parseEntityTransformParams({ fields: 'kind' })!(entity)).toEqual({ kind: 'k', }); - expect(translateQueryToFieldMapper({ fields: ['kind'] })(entity)).toEqual({ + expect(parseEntityTransformParams({ fields: ['kind'] })!(entity)).toEqual({ kind: 'k', }); expect( - translateQueryToFieldMapper({ fields: ['kind', 'apiVersion'] })(entity), + parseEntityTransformParams({ fields: ['kind', 'apiVersion'] })!(entity), ).toEqual({ apiVersion: 'av', kind: 'k' }); }); it('supports sub-selection properly', () => { expect( - translateQueryToFieldMapper({ fields: 'kind,metadata.name' })(entity), + parseEntityTransformParams({ fields: 'kind,metadata.name' })!(entity), ).toEqual({ kind: 'k', metadata: { name: 'n' } }); expect( - translateQueryToFieldMapper({ fields: 'metadata' })(entity), + parseEntityTransformParams({ fields: 'metadata' })!(entity), ).toEqual({ metadata: { name: 'n', tags: ['t1', 't2'] } }); }); }); diff --git a/plugins/catalog-backend/src/service/filterQuery.ts b/plugins/catalog-backend/src/service/request/parseEntityTransformParams.ts similarity index 66% rename from plugins/catalog-backend/src/service/filterQuery.ts rename to plugins/catalog-backend/src/service/request/parseEntityTransformParams.ts index 5e6b18ddf1..0a7c97e8dc 100644 --- a/plugins/catalog-backend/src/service/filterQuery.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityTransformParams.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,26 +14,18 @@ * limitations under the License. */ -import { InputError } from '@backstage/errors'; import { Entity } from '@backstage/catalog-model'; +import { InputError } from '@backstage/errors'; import lodash from 'lodash'; -import { RecursivePartial } from '../util'; +import { RecursivePartial } from '../../util'; +import { parseStringsParam } from './common'; -type FieldMapper = (entity: Entity) => Entity; - -export function translateQueryToFieldMapper( - query: Record, -): FieldMapper { - if (!query.fields) { - return x => x; - } - - const fieldsStrings = [query.fields].flat() as string[]; - - if (fieldsStrings.some(s => typeof s !== 'string')) { - throw new InputError( - 'Only string type fields query parameters are supported', - ); +export function parseEntityTransformParams( + params: Record, +): ((entity: Entity) => Entity) | undefined { + const fieldsStrings = parseStringsParam(params.fields, 'fields'); + if (!fieldsStrings) { + return undefined; } const fields = fieldsStrings @@ -43,13 +35,11 @@ export function translateQueryToFieldMapper( .filter(Boolean); if (!fields.length) { - return x => x; + return undefined; } if (fields.some(f => f.includes('['))) { - throw new InputError( - 'Array type fields query parameters are not supported', - ); + throw new InputError('invalid fields, array type fields are not supported'); } return input => { diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index f7655b859e..a49f57da92 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -22,8 +22,8 @@ import request from 'supertest'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { LocationResponse } from '../catalog/types'; import { HigherOrderOperation } from '../ingestion/types'; -import { EntityFilters } from './EntityFilters'; import { createRouter } from './router'; +import { basicEntityFilter } from './request'; describe('createRouter', () => { let entitiesCatalog: jest.Mocked; @@ -69,7 +69,10 @@ describe('createRouter', () => { { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - entitiesCatalog.entities.mockResolvedValueOnce(entities); + entitiesCatalog.entities.mockResolvedValueOnce({ + entities: [entities[0]], + pageInfo: { hasNext: false }, + }); const response = await request(app).get('/entities'); @@ -78,7 +81,10 @@ describe('createRouter', () => { }); it('parses single and multiple request parameters and passes them down', async () => { - entitiesCatalog.entities.mockResolvedValueOnce([]); + entitiesCatalog.entities.mockResolvedValueOnce({ + entities: [], + pageInfo: { hasNext: false }, + }); const response = await request(app).get( '/entities?filter=a=1,a=2,b=3&filter=c=4', ); @@ -86,15 +92,17 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); expect(entitiesCatalog.entities).toHaveBeenCalledWith({ - anyOf: [ - { - allOf: [ - { key: 'a', matchValueIn: ['1', '2'] }, - { key: 'b', matchValueIn: ['3'] }, - ], - }, - { allOf: [{ key: 'c', matchValueIn: ['4'] }] }, - ], + filter: { + anyOf: [ + { + allOf: [ + { key: 'a', matchValueIn: ['1', '2'] }, + { key: 'b', matchValueIn: ['3'] }, + ], + }, + { allOf: [{ key: 'c', matchValueIn: ['4'] }] }, + ], + }, }); }); }); @@ -108,27 +116,33 @@ describe('createRouter', () => { name: 'c', }, }; - entitiesCatalog.entities.mockResolvedValue([entity]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [entity], + pageInfo: { hasNext: false }, + }); const response = await request(app).get('/entities/by-uid/zzz'); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith( - EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }), - ); + expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + filter: basicEntityFilter({ 'metadata.uid': 'zzz' }), + }); expect(response.status).toEqual(200); expect(response.body).toEqual(expect.objectContaining(entity)); }); it('responds with a 404 for missing entities', async () => { - entitiesCatalog.entities.mockResolvedValue([]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [], + pageInfo: { hasNext: false }, + }); const response = await request(app).get('/entities/by-uid/zzz'); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith( - EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }), - ); + expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + filter: basicEntityFilter({ 'metadata.uid': 'zzz' }), + }); expect(response.status).toEqual(404); expect(response.text).toMatch(/uid/); }); @@ -144,35 +158,41 @@ describe('createRouter', () => { namespace: 'ns', }, }; - entitiesCatalog.entities.mockResolvedValue([entity]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [entity], + pageInfo: { hasNext: false }, + }); const response = await request(app).get('/entities/by-name/k/ns/n'); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith( - EntityFilters.ofMatchers({ + expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + filter: basicEntityFilter({ kind: 'k', 'metadata.namespace': 'ns', 'metadata.name': 'n', }), - ); + }); expect(response.status).toEqual(200); expect(response.body).toEqual(expect.objectContaining(entity)); }); it('responds with a 404 for missing entities', async () => { - entitiesCatalog.entities.mockResolvedValue([]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [], + pageInfo: { hasNext: false }, + }); const response = await request(app).get('/entities/by-name/b/d/c'); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith( - EntityFilters.ofMatchers({ + expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + filter: basicEntityFilter({ kind: 'b', 'metadata.namespace': 'd', 'metadata.name': 'c', }), - ); + }); expect(response.status).toEqual(404); expect(response.text).toMatch(/name/); }); @@ -203,7 +223,10 @@ describe('createRouter', () => { entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([ { entityId: 'u' }, ]); - entitiesCatalog.entities.mockResolvedValue([entity]); + entitiesCatalog.entities.mockResolvedValue({ + entities: [entity], + pageInfo: { hasNext: false }, + }); const response = await request(app) .post('/entities') @@ -215,9 +238,9 @@ describe('createRouter', () => { { entity, relations: [] }, ]); expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith( - EntityFilters.ofMatchers({ 'metadata.uid': 'u' }), - ); + expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + filter: basicEntityFilter({ 'metadata.uid': 'u' }), + }); expect(response.status).toEqual(200); expect(response.body).toEqual(entity); }); diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 96e424c70b..c10604636d 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -27,8 +27,12 @@ import { Logger } from 'winston'; import yn from 'yn'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types'; -import { EntityFilters } from './EntityFilters'; -import { translateQueryToFieldMapper } from './filterQuery'; +import { + basicEntityFilter, + parseEntityFilterParams, + parseEntityPaginationParams, + parseEntityTransformParams, +} from './request'; import { requireRequestBody, validateRequestBody } from './util'; export interface RouterOptions { @@ -55,10 +59,22 @@ export async function createRouter( if (entitiesCatalog) { router .get('/entities', async (req, res) => { - const filter = EntityFilters.ofQuery(req.query); - const fieldMapper = translateQueryToFieldMapper(req.query); - const entities = await entitiesCatalog.entities(filter); - res.status(200).json(entities.map(fieldMapper)); + const { entities, pageInfo } = await entitiesCatalog.entities({ + filter: parseEntityFilterParams(req.query), + fields: parseEntityTransformParams(req.query), + pagination: parseEntityPaginationParams(req.query), + }); + + // Add a Link header to the next page + if (pageInfo.hasNext) { + const url = new URL(`http://ignored${req.url}`); + url.searchParams.delete('offset'); + url.searchParams.set('after', pageInfo.endCursor); + res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`); + } + + // TODO(freben): encode the pageInfo in the response + res.json(entities); }) .post('/entities', async (req, res) => { /* @@ -75,16 +91,16 @@ export async function createRouter( const [result] = await entitiesCatalog.batchAddOrUpdateEntities([ { entity: body as Entity, relations: [] }, ]); - const [entity] = await entitiesCatalog.entities( - EntityFilters.ofMatchers({ 'metadata.uid': result.entityId }), - ); - res.status(200).json(entity); + const response = await entitiesCatalog.entities({ + filter: basicEntityFilter({ 'metadata.uid': result.entityId }), + }); + res.status(200).json(response.entities[0]); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; - const entities = await entitiesCatalog.entities( - EntityFilters.ofMatchers({ 'metadata.uid': uid }), - ); + const { entities } = await entitiesCatalog.entities({ + filter: basicEntityFilter({ 'metadata.uid': uid }), + }); if (!entities.length) { throw new NotFoundError(`No entity with uid ${uid}`); } @@ -97,13 +113,13 @@ export async function createRouter( }) .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => { const { kind, namespace, name } = req.params; - const entities = await entitiesCatalog.entities( - EntityFilters.ofMatchers({ + const { entities } = await entitiesCatalog.entities({ + filter: basicEntityFilter({ kind: kind, 'metadata.namespace': namespace, 'metadata.name': name, }), - ); + }); if (!entities.length) { throw new NotFoundError( `No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,