diff --git a/.changeset/chilly-frogs-kiss.md b/.changeset/chilly-frogs-kiss.md new file mode 100644 index 0000000000..bd1c7c7f1b --- /dev/null +++ b/.changeset/chilly-frogs-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Support filtering entities via property existence. For example you can now query with `/entities?filter=metadata.annotations.blah` to fetch all entities that has the particular property defined. diff --git a/.changeset/five-dolls-complain.md b/.changeset/five-dolls-complain.md new file mode 100644 index 0000000000..1d614019a3 --- /dev/null +++ b/.changeset/five-dolls-complain.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Export `CATALOG_FILTER_EXISTS` symbol diff --git a/.changeset/lucky-boats-rescue.md b/.changeset/lucky-boats-rescue.md new file mode 100644 index 0000000000..748f0cccb3 --- /dev/null +++ b/.changeset/lucky-boats-rescue.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +Support filtering entities via property existence diff --git a/.changeset/selfish-phones-double.md b/.changeset/selfish-phones-double.md new file mode 100644 index 0000000000..ad4880c983 --- /dev/null +++ b/.changeset/selfish-phones-double.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Optimize load times by only fetching entities with the `backstage.io/techdocs-ref` annotation diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index c356deed4f..173af9aef3 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -25,6 +25,11 @@ export type AddLocationResponse = { entities: Entity[]; }; +// Warning: (ae-missing-release-tag) "CATALOG_FILTER_EXISTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const CATALOG_FILTER_EXISTS: unique symbol; + // Warning: (ae-missing-release-tag) "CatalogApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -123,8 +128,8 @@ export class CatalogClient implements CatalogApi { // @public (undocumented) export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 4e3d95f99f..d2db98bb25 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { CatalogListResponse } from './types/api'; +import { CATALOG_FILTER_EXISTS, CatalogListResponse } from './types/api'; import { DiscoveryApi } from './types/discovery'; const server = setupServer(); @@ -83,7 +83,7 @@ describe('CatalogClient', () => { server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { expect(req.url.search).toBe( - '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2', + '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', ); return res(ctx.json([])); }), @@ -100,6 +100,9 @@ describe('CatalogClient', () => { { a: '2', }, + { + c: CATALOG_FILTER_EXISTS, + }, ], }, { token }, @@ -113,7 +116,7 @@ describe('CatalogClient', () => { server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { - expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D'); + expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D,c'); return res(ctx.json([])); }), ); @@ -124,6 +127,7 @@ describe('CatalogClient', () => { a: '1', b: ['2', '3'], รถ: '=', + c: CATALOG_FILTER_EXISTS, }, }, { token }, diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 5dfcddde0c..ef90287c37 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -26,6 +26,7 @@ import { import { ResponseError } from '@backstage/errors'; import fetch from 'cross-fetch'; import { + CATALOG_FILTER_EXISTS, AddLocationRequest, AddLocationResponse, CatalogApi, @@ -69,9 +70,13 @@ export class CatalogClient implements CatalogApi { const filterParts: string[] = []; for (const [key, value] of Object.entries(filterItem)) { for (const v of [value].flat()) { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } } } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index ae3fd7b514..ab5193e38e 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -16,10 +16,12 @@ import { Entity, EntityName, Location } from '@backstage/catalog-model'; +export const CATALOG_FILTER_EXISTS = Symbol(); + export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 6ec17fbae1..3bfcdd7fba 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -22,4 +22,5 @@ export type { CatalogListResponse, CatalogRequestOptions, } from './api'; +export { CATALOG_FILTER_EXISTS } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 0bf7831c6d..b6f2df602a 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -636,6 +636,7 @@ export type EntitiesCatalog = { export type EntitiesSearchFilter = { key: string; matchValueIn?: string[]; + matchValueExists?: boolean; }; // Warning: (ae-missing-release-tag) "entity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -1077,16 +1078,16 @@ export class UrlReaderProcessor implements CatalogProcessor { // src/catalog/types.d.ts:44:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/catalog/types.d.ts:45:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/catalog/types.d.ts:75:5 - (ae-forgotten-export) The symbol "LocationUpdateLogEvent" needs to be exported by the entry point index.d.ts -// src/database/types.d.ts:121:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:127:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:128:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:142:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:143:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:144:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/database/types.d.ts:146:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:159:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:160:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:161:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:147:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:148:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:150:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:163:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:164:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/database/types.d.ts:165:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts // src/ingestion/processors/types.d.ts:7:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/ingestion/processors/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index 2dc3d91465..f459cbc446 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -491,6 +491,98 @@ describe('CommonDatabase', () => { ]), ); }); + + it('can get all specific entities for matching existence filters', async () => { + const entities: Entity[] = [ + { + apiVersion: 'A', + kind: 'K1', + metadata: { + name: 'N', + annotations: { + foo: 'bar', + }, + }, + spec: { c: 'SOME' }, + }, + { + apiVersion: 'a', + kind: 'k2', + metadata: { + name: 'N', + annotations: { + foo: 'bar', + }, + }, + spec: { c: 'Some' }, + }, + { + apiVersion: 'a', + kind: 'k3', + metadata: { name: 'n' }, + spec: { c: 'somE' }, + }, + ]; + + await db.transaction(async tx => { + await db.addEntities( + tx, + entities.map(entity => ({ entity, relations: [] })), + ); + }); + + const existRows = await db.transaction(async tx => + db.entities(tx, { + filter: { + anyOf: [ + { + allOf: [ + { key: 'metadata.annotations.foo', matchValueExists: true }, + ], + }, + ], + }, + }), + ); + + expect(existRows.entities.length).toEqual(2); + expect(existRows.entities).toEqual( + expect.arrayContaining([ + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'K1' }), + }, + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'k2' }), + }, + ]), + ); + + const nonExistRows = await db.transaction(async tx => + db.entities(tx, { + filter: { + anyOf: [ + { + allOf: [ + { key: 'metadata.annotations.foo', matchValueExists: false }, + ], + }, + ], + }, + }), + ); + + expect(nonExistRows.entities.length).toEqual(1); + expect(nonExistRows.entities).toEqual( + expect.arrayContaining([ + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'k3' }), + }, + ]), + ); + }); }); describe('setRelations', () => { diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index b4f76d5cbc..272ac08c4c 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -218,7 +218,11 @@ export class CommonDatabase implements Database { for (const singleFilter of request?.filter?.anyOf ?? []) { entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() { - for (const { key, matchValueIn } of singleFilter.allOf) { + for (const { + key, + matchValueIn, + matchValueExists, + } of singleFilter.allOf) { // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. @@ -226,7 +230,7 @@ export class CommonDatabase implements Database { .select('entity_id') .where(function keyFilter() { this.andWhere({ key: key.toLowerCase() }); - if (matchValueIn) { + if (matchValueExists !== false && matchValueIn) { if (matchValueIn.length === 1) { this.andWhere({ value: matchValueIn[0].toLowerCase() }); } else if (matchValueIn.length > 1) { @@ -238,7 +242,12 @@ export class CommonDatabase implements Database { } } }); - this.andWhere('id', 'in', matchQuery); + // Explicitly evaluate matchValueExists as a boolean since it may be undefined + this.andWhere( + 'id', + matchValueExists === false ? 'not in' : 'in', + matchQuery, + ); } }); } diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 3a3bbb7f1e..e1a253decc 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -118,6 +118,11 @@ export type EntitiesSearchFilter = { * case insensitive. */ matchValueIn?: string[]; + + /** + * Match on existence of key. + */ + matchValueExists?: boolean; }; /** diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index f3921602e2..7cc99210c9 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -78,7 +78,11 @@ export class NextEntitiesCatalog implements EntitiesCatalog { for (const singleFilter of request?.filter?.anyOf ?? []) { entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() { - for (const { key, matchValueIn } of singleFilter.allOf) { + for (const { + key, + matchValueIn, + matchValueExists, + } of singleFilter.allOf) { // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. @@ -86,7 +90,7 @@ export class NextEntitiesCatalog implements EntitiesCatalog { .select('entity_id') .where(function keyFilter() { this.andWhere({ key: key.toLowerCase() }); - if (matchValueIn) { + if (matchValueExists !== false && matchValueIn) { if (matchValueIn.length === 1) { this.andWhere({ value: matchValueIn[0].toLowerCase() }); } else if (matchValueIn.length > 1) { @@ -98,7 +102,12 @@ export class NextEntitiesCatalog implements EntitiesCatalog { } } }); - this.andWhere('entity_id', 'in', matchQuery); + // Explicitly evaluate matchValueExists as a boolean since it may be undefined + this.andWhere( + 'entity_id', + matchValueExists === false ? 'not in' : 'in', + matchQuery, + ); } }); } diff --git a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts index e2b953ddef..fc22a638de 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts @@ -69,9 +69,11 @@ describe('parseEntityFilterParams', () => { describe('parseEntityFilterString', () => { it('works for the happy path', () => { expect(parseEntityFilterString('')).toBeUndefined(); - expect(parseEntityFilterString('a=1,b=2,a=3')).toEqual([ + expect(parseEntityFilterString('a=1,b=2,a=3,c,d=')).toEqual([ { key: 'a', matchValueIn: ['1', '3'] }, { key: 'b', matchValueIn: ['2'] }, + { key: 'c', matchValueExists: true }, + { key: 'd', matchValueIn: [''] }, ]); }); @@ -83,17 +85,11 @@ describe('parseEntityFilterString', () => { }); it('rejects malformed strings', () => { - expect(() => parseEntityFilterString('x=2,a=')).toThrow( - "Invalid filter, 'a=' is not a valid statement (expected a string on the form a=b)", - ); expect(() => parseEntityFilterString('x=2,=a')).toThrow( - "Invalid filter, '=a' is not a valid statement (expected a string on the form a=b)", + "Invalid filter, '=a' is not a valid statement (expected a string on the form a=b or a= or a)", ); expect(() => parseEntityFilterString('x=2,=')).toThrow( - "Invalid filter, '=' is not a valid statement (expected a string on the form a=b)", - ); - expect(() => parseEntityFilterString('x=2,a')).toThrow( - "Invalid filter, 'a' is not a valid statement (expected a string on the form a=b)", + "Invalid filter, '=' is not a valid statement (expected a string on the form a=b or a= or a)", ); }); }); diff --git a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts index 154a25c541..c63b366bc7 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts @@ -61,25 +61,26 @@ export function parseEntityFilterString( for (const statement of statements) { const equalsIndex = statement.indexOf('='); - if (equalsIndex < 1) { - throw new InputError( - `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`, - ); - } - const key = statement.substr(0, equalsIndex).trim(); - const value = statement.substr(equalsIndex + 1).trim(); - if (!key || !value) { + const key = + equalsIndex === -1 ? statement : statement.substr(0, equalsIndex).trim(); + const value = + equalsIndex === -1 ? undefined : statement.substr(equalsIndex + 1).trim(); + if (!key) { throw new InputError( - `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`, + `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b or a= or a)`, ); } const f = - key in filtersByKey - ? filtersByKey[key] - : (filtersByKey[key] = { key, matchValueIn: [] }); - f.matchValueIn!.push(value); + key in filtersByKey ? filtersByKey[key] : (filtersByKey[key] = { key }); + + if (value === undefined) { + f.matchValueExists = true; + } else { + f.matchValueIn = f.matchValueIn || []; + f.matchValueIn.push(value); + } } return Object.values(filtersByKey); diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 2555cc7c0c..01c88be089 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -7,6 +7,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { AsyncState } from 'react-use/lib/useAsync'; +import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { CatalogApi } from '@backstage/catalog-client'; import { ComponentEntity } from '@backstage/catalog-model'; import { Context } from 'react'; @@ -22,6 +23,8 @@ import { SystemEntity } from '@backstage/catalog-model'; import { TableColumn } from '@backstage/core-components'; import { UserEntity } from '@backstage/catalog-model'; +export { CATALOG_FILTER_EXISTS }; + export { CatalogApi }; // Warning: (ae-missing-release-tag) "catalogApiRef" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/catalog-react/src/index.ts b/plugins/catalog-react/src/index.ts index 4ee613012d..4e005c0cdc 100644 --- a/plugins/catalog-react/src/index.ts +++ b/plugins/catalog-react/src/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ export type { CatalogApi } from '@backstage/catalog-client'; +export { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; export { catalogApiRef } from './api'; export * from './components'; export * from './hooks'; diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx index c5d05f1474..649d69ed8a 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx @@ -19,6 +19,7 @@ import { useAsync } from 'react-use'; import { makeStyles } from '@material-ui/core'; import { CSSProperties } from '@material-ui/styles'; import { + CATALOG_FILTER_EXISTS, catalogApiRef, CatalogApi, isOwnerOf, @@ -125,6 +126,9 @@ export const TechDocsCustomHome = ({ const { value: entities, loading, error } = useAsync(async () => { const response = await catalogApi.getEntities({ + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': CATALOG_FILTER_EXISTS, + }, fields: [ 'apiVersion', 'kind',