paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Introduce paging in the catalog /entities endpoint

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2021-03-05 19:33:06 +01:00
parent 285445053a
commit c862b3f36f
23 changed files with 805 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/twelve-files-invite.md
+32
View File
@@ -0,0 +1,32 @@
---
'@backstage/plugin-catalog-backend': patch
---
Introduce pagination in the /entities catalog endpoint.
Pagination is requested using query parameters. Currently supported parameters, all optional, are:
- `limit` - an integer number of entities to return, at most
- `offset` - an integer number of entities to skip over at the start
- `after` - an opaque string cursor as returned by a previous paginated request
Example request:
`GET /entities?limit=100`
Example response:
```
200 OK
Content-Type: application/json; charset=utf-8
Link: </entities?limit=100&after=eyJsaW1pdCI6Miwib2Zmc2V0IjoyfQ%3D%3D>; rel="next"
<more headers>
[{"metadata":{...
```
Note the Link header. It contains the URL (path and query part, relative to the catalog root) to use for requesting the next page.
It uses the `after` cursor to point out the end of the previous page. If the Link header is not present, there is no more data to read.
The current implementation is naive and encodes offset/limit in the cursor implementation, so it is not robust in the face of overlapping
changes to the catalog. This can be improved separately in the future without having to change the calling patterns.
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts
+68 -41
View File
@@ -17,7 +17,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
import { Database, DatabaseManager, Transaction } from '../database';
import { EntityFilters } from '../service/EntityFilters';
import { basicEntityFilter } from '../service/request';
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
import { EntityUpsertRequest } from './types';
@@ -63,7 +63,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([]);
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -74,12 +77,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
@@ -96,7 +100,10 @@ describe('DatabaseEntitiesCatalog', () => {
namespace: 'd',
},
};
db.entities.mockResolvedValue([]);
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -108,12 +115,13 @@ describe('DatabaseEntitiesCatalog', () => {
);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
@@ -147,7 +155,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
},
};
db.entities.mockResolvedValue([{ entity: dbEntity }]);
db.entities.mockResolvedValue({
entities: [{ entity: dbEntity }],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -198,7 +209,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([existing]);
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNext: false },
});
db.entityByUid.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue({ entity });
@@ -208,12 +222,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).toHaveBeenCalledTimes(1);
expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u');
@@ -272,7 +287,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([existing]);
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNext: false },
});
db.entityByName.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue(existing);
@@ -282,12 +300,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).toHaveBeenCalledTimes(1);
expect(db.entityByName).toHaveBeenCalledWith(transaction, {
kind: 'b',
@@ -334,7 +353,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([{ entity }]);
db.entities.mockResolvedValue({
entities: [{ entity }],
pageInfo: { hasNext: false },
});
db.entityByUid.mockResolvedValue({ entity });
db.updateEntity.mockResolvedValue({ entity });
@@ -344,12 +366,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).not.toHaveBeenCalled();
expect(db.updateEntity).not.toHaveBeenCalled();
@@ -377,7 +400,7 @@ describe('DatabaseEntitiesCatalog', () => {
await catalog.batchAddOrUpdateEntities(entities);
const afterFirst = await catalog.entities();
expect(afterFirst.length).toBe(300);
expect(afterFirst.entities.length).toBe(300);
entities[40].entity.metadata.op = 'changed';
entities.push({
@@ -391,9 +414,13 @@ describe('DatabaseEntitiesCatalog', () => {
await catalog.batchAddOrUpdateEntities(entities);
const afterSecond = await catalog.entities();
expect(afterSecond.length).toBe(301);
expect(afterSecond.find(e => e.metadata.op === 'changed')).toBeDefined();
expect(afterSecond.find(e => e.metadata.op === 'added')).toBeDefined();
expect(afterSecond.entities.length).toBe(301);
expect(
afterSecond.entities.find(e => e.metadata.op === 'changed'),
).toBeDefined();
expect(
afterSecond.entities.find(e => e.metadata.op === 'added'),
).toBeDefined();
}, 10000);
});
});
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+42 -28
View File
@@ -14,7 +14,6 @@
* limitations under the License.
*/
import { ConflictError, NotFoundError } from '@backstage/errors';
import {
Entity,
entityHasChanges,
@@ -23,19 +22,18 @@ import {
LOCATION_ANNOTATION,
serializeEntityRef,
} from '@backstage/catalog-model';
import { ConflictError, NotFoundError } from '@backstage/errors';
import { chunk, groupBy } from 'lodash';
import limiterFactory from 'p-limit';
import { Logger } from 'winston';
import type {
Database,
DbEntityResponse,
EntityFilter,
Transaction,
} from '../database';
import { EntityFilters } from '../service/EntityFilters';
import type { Database, DbEntityResponse, Transaction } from '../database';
import { DbEntitiesRequest } from '../database/types';
import { basicEntityFilter } from '../service/request';
import { durationText } from '../util/timing';
import type {
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
EntityUpsertRequest,
EntityUpsertResponse,
} from './types';
@@ -65,11 +63,24 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
private readonly logger: Logger,
) {}
async entities(filter?: EntityFilter): Promise<Entity[]> {
const items = await this.database.transaction(tx =>
this.database.entities(tx, filter),
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
const dbRequest: DbEntitiesRequest = {
filter: request?.filter,
pagination: request?.pagination,
};
const dbResponse = await this.database.transaction(tx =>
this.database.entities(tx, dbRequest),
);
return items.map(i => i.entity);
const entities = dbResponse.entities.map(e =>
request?.fields ? request.fields(e.entity) : e.entity,
);
return {
entities,
pageInfo: dbResponse.pageInfo,
};
}
async removeEntityByUid(uid: string): Promise<void> {
@@ -78,16 +89,20 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
if (!entityResponse) {
throw new NotFoundError(`Entity with ID ${uid} was not found`);
}
const location =
entityResponse.entity.metadata.annotations?.[LOCATION_ANNOTATION];
const colocatedEntities = location
? await this.database.entities(
tx,
EntityFilters.ofMatchers({
[`metadata.annotations.${LOCATION_ANNOTATION}`]: location,
}),
)
? (
await this.database.entities(tx, {
filter: basicEntityFilter({
[`metadata.annotations.${LOCATION_ANNOTATION}`]: location,
}),
})
).entities
: [entityResponse];
for (const dbResponse of colocatedEntities) {
await this.database.removeEntityByUid(
tx,
@@ -98,6 +113,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
if (entityResponse.locationId) {
await this.database.removeLocation(tx, entityResponse?.locationId!);
}
return undefined;
});
}
@@ -206,13 +222,12 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
}
if (options?.outputEntities && responses.length > 0) {
const writtenEntities = await this.database.entities(
tx,
EntityFilters.ofMatchers({
const writtenEntities = await this.database.entities(tx, {
filter: basicEntityFilter({
'metadata.uid': responses.map(e => e.entityId),
}),
);
responses = writtenEntities.map(e => ({
});
responses = writtenEntities.entities.map(e => ({
entityId: e.entity.metadata.uid!,
entity: e.entity,
}));
@@ -247,17 +262,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
// Here we make use of the fact that all of the entities share kind and
// namespace within a batch
const names = requests.map(({ entity }) => entity.metadata.name);
const oldEntities = await this.database.entities(
tx,
EntityFilters.ofMatchers({
const oldEntitiesResponse = await this.database.entities(tx, {
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': names,
}),
);
});
const oldEntitiesByName = new Map(
oldEntities.map(e => [e.entity.metadata.name, e.entity]),
oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]),
);
const toAdd: EntityUpsertRequest[] = [];
plugins/catalog-backend/src/catalog/types.ts
+23 -3
View File
@@ -15,12 +15,32 @@
*/
import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
import type { EntityFilter } from '../database';
import { EntityFilter, EntityPagination } from '../database/types';
//
// Entities
//
export type PageInfo =
| {
hasNext: false;
}
| {
hasNext: true;
endCursor: string;
};
export type EntitiesRequest = {
filter?: EntityFilter;
fields?: (entity: Entity) => Entity;
pagination?: EntityPagination;
};
export type EntitiesResponse = {
entities: Entity[];
pageInfo: PageInfo;
};
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
@@ -35,9 +55,9 @@ export type EntitiesCatalog = {
/**
* Fetch entities.
*
* @param filter A filter to apply when reading
* @param request Request options
*/
entities(filter?: EntityFilter): Promise<Entity[]>;
entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
/**
* Removes a single entity.
plugins/catalog-backend/src/database/CommonDatabase.test.ts
+20 -18
View File
@@ -14,9 +14,9 @@
* limitations under the License.
*/
import { ConflictError } from '@backstage/errors';
import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
import { EntityFilters } from '../service/EntityFilters';
import { ConflictError } from '@backstage/errors';
import { basicEntityFilter } from '../service/request';
import { DatabaseManager } from './DatabaseManager';
import type {
DbEntityRequest,
@@ -385,8 +385,8 @@ describe('CommonDatabase', () => {
]);
});
const result = await db.transaction(async tx => db.entities(tx));
expect(result.length).toEqual(2);
expect(result).toEqual(
expect(result.entities.length).toEqual(2);
expect(result.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
@@ -424,11 +424,13 @@ describe('CommonDatabase', () => {
);
});
await expect(
db.transaction(async tx =>
db.entities(tx, EntityFilters.ofFilterString('kind=k2,spec.c=some')),
),
).resolves.toEqual([
const response = await db.transaction(async tx =>
db.entities(tx, {
filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }),
}),
);
expect(response.entities).toEqual([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
@@ -466,14 +468,13 @@ describe('CommonDatabase', () => {
});
const rows = await db.transaction(async tx =>
db.entities(
tx,
EntityFilters.ofFilterString('ApiVersioN=A,spEc.C=some'),
),
db.entities(tx, {
filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }),
}),
);
expect(rows.length).toEqual(3);
expect(rows).toEqual(
expect(rows.entities.length).toEqual(3);
expect(rows.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
@@ -531,7 +532,8 @@ describe('CommonDatabase', () => {
{ target: mockRelations[0].target, type: 'child' },
]);
const [returnedEntity3] = await db.transaction(tx => db.entities(tx));
const { entities } = await db.transaction(tx => db.entities(tx));
const [returnedEntity3] = entities;
expect(returnedEntity3?.entity.relations).toEqual([
{ target: mockRelations[0].target, type: 'child' },
]);
@@ -615,7 +617,7 @@ describe('CommonDatabase', () => {
const res = await db.transaction(tx => db.entities(tx));
expect(
res.map(r => ({
res.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
@@ -660,7 +662,7 @@ describe('CommonDatabase', () => {
const res2 = await db.transaction(tx => db.entities(tx));
expect(
res2.map(r => ({
res2.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
plugins/catalog-backend/src/database/CommonDatabase.ts
+76 -7
View File
@@ -35,13 +35,16 @@ import {
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
DbEntitiesRelationsRow,
DbEntitiesRequest,
DbEntitiesResponse,
DbEntitiesRow,
DbEntitiesSearchRow,
DbEntityRequest,
DbEntityResponse,
DbLocationsRow,
DbLocationsRowWithStatus,
EntityFilter,
DbPageInfo,
EntityPagination,
Transaction,
} from './types';
@@ -207,13 +210,13 @@ export class CommonDatabase implements Database {
async entities(
txOpaque: Transaction,
filter?: EntityFilter,
): Promise<DbEntityResponse[]> {
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse> {
const tx = txOpaque as Knex.Transaction;
let entitiesQuery = tx<DbEntitiesRow>('entities');
for (const singleFilter of filter?.anyOf ?? []) {
for (const singleFilter of request?.filter?.anyOf ?? []) {
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
for (const { key, matchValueIn } of singleFilter.allOf) {
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
@@ -235,17 +238,43 @@ export class CommonDatabase implements Database {
}
}
});
this.andWhere('id', 'in', matchQuery);
}
});
}
const rows = await entitiesQuery
entitiesQuery = entitiesQuery
.select('entities.*')
.orderBy('full_name', 'asc');
return this.toEntityResponses(tx, rows);
const { limit, offset } = parsePagination(request?.pagination);
if (limit !== undefined) {
entitiesQuery = entitiesQuery.limit(limit + 1);
}
if (offset !== undefined) {
entitiesQuery = entitiesQuery.offset(offset);
}
let rows = await entitiesQuery;
let pageInfo: DbPageInfo;
if (limit === undefined || rows.length <= limit) {
pageInfo = { hasNext: false };
} else {
rows = rows.slice(0, -1);
pageInfo = {
hasNext: true,
endCursor: stringifyPagination({
limit,
offset: (offset ?? 0) + limit,
}),
};
}
return {
entities: await this.toEntityResponses(tx, rows),
pageInfo,
};
}
async entityByName(
@@ -519,6 +548,46 @@ export class CommonDatabase implements Database {
}
}
function parsePagination(
input?: EntityPagination,
): { limit?: number; offset?: number } {
if (!input) {
return {};
}
let { limit, offset } = input;
if (input.after !== undefined) {
let cursor;
try {
const json = Buffer.from(input.after, 'base64').toString('utf8');
cursor = JSON.parse(json);
} catch {
throw new InputError('Malformed after cursor');
}
if (cursor.limit !== undefined) {
if (!Number.isInteger(cursor.limit)) {
throw new InputError('Malformed after cursor');
}
limit = cursor.limit;
}
if (cursor.offset !== undefined) {
if (!Number.isInteger(cursor.offset)) {
throw new InputError('Malformed after cursor');
}
offset = cursor.offset;
}
}
return { limit, offset };
}
function stringifyPagination(input: { limit: number; offset: number }) {
const json = JSON.stringify({ limit: input.limit, offset: input.offset });
const base64 = Buffer.from(json, 'utf8').toString('base64');
return base64;
}
function deduplicateRelations(
rows: DbEntitiesRelationsRow[],
): DbEntitiesRelationsRow[] {
plugins/catalog-backend/src/database/index.ts
+1
View File
@@ -22,5 +22,6 @@ export type {
DbEntityResponse,
EntitiesSearchFilter,
EntityFilter,
EntityPagination,
Transaction,
} from './types';
plugins/catalog-backend/src/database/types.ts
+32 -1
View File
@@ -36,6 +36,25 @@ export type DbEntityRequest = {
relations: EntityRelationSpec[];
};
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
};
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
};
export type DbPageInfo =
| {
hasNext: false;
}
| {
hasNext: true;
endCursor: string;
};
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
@@ -111,6 +130,15 @@ export type EntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
/**
* A pagination rule for entities.
*/
export type EntityPagination = {
limit?: number;
offset?: number;
after?: string;
};
/**
* An abstraction for transactions of the underlying database technology.
*/
@@ -170,7 +198,10 @@ export type Database = {
matchingGeneration?: number,
): Promise<DbEntityResponse>;
entities(tx: Transaction, filter?: EntityFilter): Promise<DbEntityResponse[]>;
entities(
tx: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse>;
entityByName(
tx: Transaction,
plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
+4 -1
View File
@@ -360,7 +360,10 @@ describe('HigherOrderOperations', () => {
entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]);
await expect(
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+6 -3
View File
@@ -61,7 +61,10 @@ describe('CatalogBuilder', () => {
it('works with no changes', async () => {
const builder = new CatalogBuilder(env);
const built = await builder.build();
await expect(built.entitiesCatalog.entities()).resolves.toEqual([]);
await expect(built.entitiesCatalog.entities()).resolves.toEqual({
entities: [],
pageInfo: { hasNext: false },
});
await expect(built.locationsCatalog.locations()).resolves.toEqual([
expect.objectContaining({
data: expect.objectContaining({ type: 'bootstrap' }),
@@ -166,7 +169,7 @@ describe('CatalogBuilder', () => {
type: 'github',
target: 'https://github.com/a/b/x.yaml',
});
const entities = await entitiesCatalog.entities();
const { entities } = await entitiesCatalog.entities();
expect(entities).toEqual([
expect.objectContaining({
@@ -200,7 +203,7 @@ describe('CatalogBuilder', () => {
type: 'x',
target: 'y',
});
const entities = await entitiesCatalog.entities();
const { entities } = await entitiesCatalog.entities();
expect.assertions(3);
expect(entities).toEqual([
plugins/catalog-backend/src/service/EntityFilters.test.ts
-113
View File
@@ -1,113 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityFilters } from './EntityFilters';
describe('EntityFilters', () => {
describe('ofQuery', () => {
it('translates empty query to empty list', () => {
const result = EntityFilters.ofQuery({});
expect(result).toEqual(undefined);
});
it('supports single-string format', () => {
const result = EntityFilters.ofQuery({ filter: 'a=1' })!;
expect(result).toEqual({
anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }],
});
});
it('supports array-of-strings format', () => {
const result = EntityFilters.ofQuery({ filter: ['a=1', 'b=2'] });
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{ allOf: [{ key: 'b', matchValueIn: ['2'] }] },
],
});
});
it('throws for non-strings', () => {
expect(() => EntityFilters.ofQuery({ filter: [3] })).toThrow(/string/);
});
});
describe('ofFilterString', () => {
it('runs the happy path', () => {
const result = EntityFilters.ofFilterString('a=1,b=2');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('ignores empty', () => {
const result = EntityFilters.ofFilterString('a=1,,b=2,');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('trims', () => {
const result = EntityFilters.ofFilterString(' a = 1 ,, b=2 ,');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('merges multiple of the same key', () => {
const result = EntityFilters.ofFilterString('a=1,a=2,b=3');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
],
});
});
it('throws on missing equal sign', () => {
expect(() => EntityFilters.ofFilterString('a,b=2')).toThrow();
});
it('throws on misplaced equal sign', () => {
expect(() => EntityFilters.ofFilterString('a=,b=2')).toThrow();
});
});
});
plugins/catalog-backend/src/service/EntityFilters.ts
-120
View File
@@ -1,120 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { EntitiesSearchFilter, EntityFilter } from '../database';
/**
* A builder that assists in creating entity filter instances.
*/
export class EntityFilters {
// Builds a filter from the value of a filter=a=1,b=2 type filter string
static ofFilterString(filterString: string): EntityFilter | undefined {
const builder = new EntityFilters();
builder.addFilterString(filterString);
return builder.build();
}
// Builds a filter from an entire query params structure
static ofQuery(query: Record<string, any>): EntityFilter | undefined {
const filterStrings = [query.filter || []].flat();
if (filterStrings.some(f => typeof f !== 'string')) {
throw new InputError(
'Only string type filter query parameters are supported',
);
}
const builder = new EntityFilters();
for (const filterString of filterStrings) {
builder.addFilterString(filterString);
}
return builder.build();
}
// Builds a filter from a { key: value } matcher object
static ofMatchers(
matchers: Record<string, string | string[]>,
): EntityFilter | undefined {
const builder = new EntityFilters();
builder.addMatchers(matchers);
return builder.build();
}
private filters: EntitiesSearchFilter[][];
constructor() {
this.filters = [];
}
addFilterString(filterString: string): EntityFilters {
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
const addFilter = (key: string, value: string) => {
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(value);
};
const statements = filterString
.split(',')
.map(s => s.trim())
.filter(Boolean);
for (const statement of statements) {
const equalsIndex = statement.indexOf('=');
if (equalsIndex < 1) {
throw new InputError('Malformed filter query');
} else {
const key = statement.substr(0, equalsIndex).trim();
const value = statement.substr(equalsIndex + 1).trim();
if (!key || !value) {
throw new InputError('Malformed filter query');
}
addFilter(key, value);
}
}
this.filters.push(Object.values(filtersByKey));
return this;
}
addMatchers(matchers: Record<string, string | string[]>): EntityFilters {
const filters: EntitiesSearchFilter[] = [];
for (const [key, value] of Object.entries(matchers)) {
filters.push({ key, matchValueIn: [value].flat() });
}
if (filters.length) {
this.filters.push(filters);
}
return this;
}
build(): EntityFilter | undefined {
if (!this.filters.length) {
return undefined;
}
return { anyOf: this.filters.map(f => ({ allOf: f })) };
}
}
plugins/catalog-backend/src/service/request/basicEntityFilter.ts
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntitiesSearchFilter, EntityFilter } from '../../database';
/**
* Forms a full EntityFilter based on a single key-value(s) object.
*/
export function basicEntityFilter(
items: Record<string, string | string[]>,
): EntityFilter {
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
for (const [key, value] of Object.entries(items)) {
const values = [value].flat();
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(...values);
}
return { anyOf: [{ allOf: Object.values(filtersByKey) }] };
}
plugins/catalog-backend/src/service/request/common.ts
+79
View File
@@ -0,0 +1,79 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
/**
* Takes a single unknown parameter and makes sure that it's a string that can
* be parsed as an integer.
*/
export function parseIntegerParam(
param: unknown,
ctx: string,
): number | undefined {
if (param === undefined) {
return undefined;
}
if (typeof param !== 'string') {
throw new InputError(`Invalid ${ctx}, not an integer on string form`);
}
const parsed = parseInt(param, 10);
if (!Number.isInteger(parsed) || String(parsed) !== param) {
throw new InputError(`Invalid ${ctx}, not an integer`);
}
return parsed;
}
/**
* Takes a single unknown parameter and makes sure that it's a string.
*/
export function parseStringParam(
param: unknown,
ctx: string,
): string | undefined {
if (param === undefined) {
return undefined;
}
if (typeof param !== 'string') {
throw new InputError(`Invalid ${ctx}, not a string`);
}
return param;
}
/**
* Takes a single unknown parameter and makes sure that it's a single string or
* an array of strings, and returns as an array.
*/
export function parseStringsParam(
param: unknown,
ctx: string,
): string[] | undefined {
if (param === undefined) {
return undefined;
}
const array = [param].flat();
if (array.some(p => typeof p !== 'string')) {
throw new InputError(`Invalid ${ctx}, not a string`);
}
return array as string[];
}
plugins/catalog-backend/src/service/request/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { basicEntityFilter } from './basicEntityFilter';
export { parseEntityFilterParams } from './parseEntityFilterParams';
export { parseEntityPaginationParams } from './parseEntityPaginationParams';
export { parseEntityTransformParams } from './parseEntityTransformParams';
plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts
+99
View File
@@ -0,0 +1,99 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
parseEntityFilterParams,
parseEntityFilterString,
} from './parseEntityFilterParams';
describe('parseEntityFilterParams', () => {
it('translates empty query to empty list', () => {
const result = parseEntityFilterParams({});
expect(result).toEqual(undefined);
});
it('supports single-string format', () => {
const result = parseEntityFilterParams({ filter: 'a=1' })!;
expect(result).toEqual({
anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }],
});
});
it('supports array-of-strings format', () => {
const result = parseEntityFilterParams({
filter: ['a=1', 'b=2'],
});
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{ allOf: [{ key: 'b', matchValueIn: ['2'] }] },
],
});
});
it('merges values within each filter', () => {
const result = parseEntityFilterParams({
filter: ['a=1', 'b=2,b=3,c=4'],
});
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{
allOf: [
{ key: 'b', matchValueIn: ['2', '3'] },
{ key: 'c', matchValueIn: ['4'] },
],
},
],
});
});
it('throws for non-strings', () => {
expect(() => parseEntityFilterParams({ filter: [3] })).toThrow(/string/);
});
});
describe('parseEntityFilterString', () => {
it('works for the happy path', () => {
expect(parseEntityFilterString('')).toBeUndefined();
expect(parseEntityFilterString('a=1,b=2,a=3')).toEqual([
{ key: 'a', matchValueIn: ['1', '3'] },
{ key: 'b', matchValueIn: ['2'] },
]);
});
it('trims values', () => {
expect(parseEntityFilterString(' a = 1 , b = 2 , a = 3 ')).toEqual([
{ key: 'a', matchValueIn: ['1', '3'] },
{ key: 'b', matchValueIn: ['2'] },
]);
});
it('rejects malformed strings', () => {
expect(() => parseEntityFilterString('x=2,a=')).toThrow(
"Invalid filter, 'a=' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,=a')).toThrow(
"Invalid filter, '=a' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,=')).toThrow(
"Invalid filter, '=' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,a')).toThrow(
"Invalid filter, 'a' is not a valid statement (expected a string on the form a=b)",
);
});
});
plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { EntitiesSearchFilter, EntityFilter } from '../../database';
import { parseStringsParam } from './common';
/**
* Parses the filtering part of a query, like
* /entities?filter=metadata.namespace=default,kind=Component
*/
export function parseEntityFilterParams(
params: Record<string, unknown>,
): EntityFilter | undefined {
// Each filter string is on the form a=b,c=d
const filterStrings = parseStringsParam(params.filter, 'filter');
if (!filterStrings) {
return undefined;
}
// Outer array: "any of the inner ones"
// Inner arrays: "all of these must match"
const filters = filterStrings.map(parseEntityFilterString).filter(Boolean);
if (!filters.length) {
return undefined;
}
return { anyOf: filters.map(f => ({ allOf: f! })) };
}
/**
* Parses a single filter string as seen in a filter query, for example
* metadata.namespace=default,kind=Component
*/
export function parseEntityFilterString(
filterString: string,
): EntitiesSearchFilter[] | undefined {
const statements = filterString
.split(',')
.map(s => s.trim())
.filter(Boolean);
if (!statements.length) {
return undefined;
}
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
for (const statement of statements) {
const equalsIndex = statement.indexOf('=');
if (equalsIndex < 1) {
throw new InputError(
`Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
);
}
const key = statement.substr(0, equalsIndex).trim();
const value = statement.substr(equalsIndex + 1).trim();
if (!key || !value) {
throw new InputError(
`Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
);
}
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(value);
}
return Object.values(filtersByKey);
}
plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseEntityPaginationParams } from './parseEntityPaginationParams';
describe('parseEntityPaginationParams', () => {
it('works for the happy path', () => {
expect(parseEntityPaginationParams({})).toBeUndefined();
});
});
plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPagination } from '../../database';
import { parseIntegerParam, parseStringParam } from './common';
/**
* Parses the pagination related parameters out of a query, e.g.
* /entities?offset=100&limit=10
*/
export function parseEntityPaginationParams(
params: Record<string, unknown>,
): EntityPagination | undefined {
const offset = parseIntegerParam(params.offset, 'offset');
const limit = parseIntegerParam(params.limit, 'limit');
const after = parseStringParam(params.after, 'after');
if (offset === undefined && limit === undefined && after === undefined) {
return undefined;
}
return {
...(offset ? { offset } : {}),
...(limit ? { limit } : {}),
...(after ? { after } : {}),
};
}
plugins/catalog-backend/src/service/filterQuery.test.ts → plugins/catalog-backend/src/service/request/parseEntityTransformParams.test.ts
+16 -15
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,9 +15,9 @@
*/
import { Entity } from '@backstage/catalog-model';
import { translateQueryToFieldMapper } from './filterQuery';
import { parseEntityTransformParams } from './parseEntityTransformParams';
describe('translateQueryToFieldMapper', () => {
describe('parseEntityTransformParams', () => {
const entity: Entity = {
apiVersion: 'av',
kind: 'k',
@@ -30,37 +30,38 @@ describe('translateQueryToFieldMapper', () => {
},
};
it('passes through when no fields given', () => {
expect(translateQueryToFieldMapper({})(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [] })(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [''] })(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [','] })(entity)).toBe(entity);
it('returns undefined when no fields given', () => {
expect(parseEntityTransformParams({})).toBeUndefined();
expect(parseEntityTransformParams({ fields: '' })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [] })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [''] })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [','] })).toBeUndefined();
});
it('rejects attempts at array filtering', () => {
expect(() =>
translateQueryToFieldMapper({ fields: 'metadata.tags[0]' })(entity),
).toThrow(/array/i);
parseEntityTransformParams({ fields: 'metadata.tags[0]' })!(entity),
).toThrow(/invalid fields, array type fields are not supported/i);
});
it('accepts both strings and arrays of strings as input', () => {
expect(translateQueryToFieldMapper({ fields: 'kind' })(entity)).toEqual({
expect(parseEntityTransformParams({ fields: 'kind' })!(entity)).toEqual({
kind: 'k',
});
expect(translateQueryToFieldMapper({ fields: ['kind'] })(entity)).toEqual({
expect(parseEntityTransformParams({ fields: ['kind'] })!(entity)).toEqual({
kind: 'k',
});
expect(
translateQueryToFieldMapper({ fields: ['kind', 'apiVersion'] })(entity),
parseEntityTransformParams({ fields: ['kind', 'apiVersion'] })!(entity),
).toEqual({ apiVersion: 'av', kind: 'k' });
});
it('supports sub-selection properly', () => {
expect(
translateQueryToFieldMapper({ fields: 'kind,metadata.name' })(entity),
parseEntityTransformParams({ fields: 'kind,metadata.name' })!(entity),
).toEqual({ kind: 'k', metadata: { name: 'n' } });
expect(
translateQueryToFieldMapper({ fields: 'metadata' })(entity),
parseEntityTransformParams({ fields: 'metadata' })!(entity),
).toEqual({ metadata: { name: 'n', tags: ['t1', 't2'] } });
});
});
plugins/catalog-backend/src/service/filterQuery.ts → plugins/catalog-backend/src/service/request/parseEntityTransformParams.ts
+12 -22
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,26 +14,18 @@
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/errors';
import lodash from 'lodash';
import { RecursivePartial } from '../util';
import { RecursivePartial } from '../../util';
import { parseStringsParam } from './common';
type FieldMapper = (entity: Entity) => Entity;
export function translateQueryToFieldMapper(
query: Record<string, any>,
): FieldMapper {
if (!query.fields) {
return x => x;
}
const fieldsStrings = [query.fields].flat() as string[];
if (fieldsStrings.some(s => typeof s !== 'string')) {
throw new InputError(
'Only string type fields query parameters are supported',
);
export function parseEntityTransformParams(
params: Record<string, unknown>,
): ((entity: Entity) => Entity) | undefined {
const fieldsStrings = parseStringsParam(params.fields, 'fields');
if (!fieldsStrings) {
return undefined;
}
const fields = fieldsStrings
@@ -43,13 +35,11 @@ export function translateQueryToFieldMapper(
.filter(Boolean);
if (!fields.length) {
return x => x;
return undefined;
}
if (fields.some(f => f.includes('['))) {
throw new InputError(
'Array type fields query parameters are not supported',
);
throw new InputError('invalid fields, array type fields are not supported');
}
return input => {
plugins/catalog-backend/src/service/router.test.ts
+55 -32
View File
@@ -22,8 +22,8 @@ import request from 'supertest';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { LocationResponse } from '../catalog/types';
import { HigherOrderOperation } from '../ingestion/types';
import { EntityFilters } from './EntityFilters';
import { createRouter } from './router';
import { basicEntityFilter } from './request';
describe('createRouter', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
@@ -69,7 +69,10 @@ describe('createRouter', () => {
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
];
entitiesCatalog.entities.mockResolvedValueOnce(entities);
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [entities[0]],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities');
@@ -78,7 +81,10 @@ describe('createRouter', () => {
});
it('parses single and multiple request parameters and passes them down', async () => {
entitiesCatalog.entities.mockResolvedValueOnce([]);
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get(
'/entities?filter=a=1,a=2,b=3&filter=c=4',
);
@@ -86,15 +92,17 @@ describe('createRouter', () => {
expect(response.status).toEqual(200);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
{ allOf: [{ key: 'c', matchValueIn: ['4'] }] },
],
filter: {
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
{ allOf: [{ key: 'c', matchValueIn: ['4'] }] },
],
},
});
});
});
@@ -108,27 +116,33 @@ describe('createRouter', () => {
name: 'c',
},
};
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/uid/);
});
@@ -144,35 +158,41 @@ describe('createRouter', () => {
namespace: 'ns',
},
};
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-name/k/ns/n');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'k',
'metadata.namespace': 'ns',
'metadata.name': 'n',
}),
);
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-name/b/d/c');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
);
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/name/);
});
@@ -203,7 +223,10 @@ describe('createRouter', () => {
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{ entityId: 'u' },
]);
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app)
.post('/entities')
@@ -215,9 +238,9 @@ describe('createRouter', () => {
{ entity, relations: [] },
]);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'u' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'u' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(entity);
});
plugins/catalog-backend/src/service/router.ts
+32 -16
View File
@@ -27,8 +27,12 @@ import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types';
import { EntityFilters } from './EntityFilters';
import { translateQueryToFieldMapper } from './filterQuery';
import {
basicEntityFilter,
parseEntityFilterParams,
parseEntityPaginationParams,
parseEntityTransformParams,
} from './request';
import { requireRequestBody, validateRequestBody } from './util';
export interface RouterOptions {
@@ -55,10 +59,22 @@ export async function createRouter(
if (entitiesCatalog) {
router
.get('/entities', async (req, res) => {
const filter = EntityFilters.ofQuery(req.query);
const fieldMapper = translateQueryToFieldMapper(req.query);
const entities = await entitiesCatalog.entities(filter);
res.status(200).json(entities.map(fieldMapper));
const { entities, pageInfo } = await entitiesCatalog.entities({
filter: parseEntityFilterParams(req.query),
fields: parseEntityTransformParams(req.query),
pagination: parseEntityPaginationParams(req.query),
});
// Add a Link header to the next page
if (pageInfo.hasNext) {
const url = new URL(`http://ignored${req.url}`);
url.searchParams.delete('offset');
url.searchParams.set('after', pageInfo.endCursor);
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
}
// TODO(freben): encode the pageInfo in the response
res.json(entities);
})
.post('/entities', async (req, res) => {
/*
@@ -75,16 +91,16 @@ export async function createRouter(
const [result] = await entitiesCatalog.batchAddOrUpdateEntities([
{ entity: body as Entity, relations: [] },
]);
const [entity] = await entitiesCatalog.entities(
EntityFilters.ofMatchers({ 'metadata.uid': result.entityId }),
);
res.status(200).json(entity);
const response = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': result.entityId }),
});
res.status(200).json(response.entities[0]);
})
.get('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
const entities = await entitiesCatalog.entities(
EntityFilters.ofMatchers({ 'metadata.uid': uid }),
);
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': uid }),
});
if (!entities.length) {
throw new NotFoundError(`No entity with uid ${uid}`);
}
@@ -97,13 +113,13 @@ export async function createRouter(
})
.get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
const { kind, namespace, name } = req.params;
const entities = await entitiesCatalog.entities(
EntityFilters.ofMatchers({
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': name,
}),
);
});
if (!entities.length) {
throw new NotFoundError(
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,