diff --git a/.changeset/strange-rocks-hide.md b/.changeset/strange-rocks-hide.md new file mode 100644 index 0000000000..e171a277fb --- /dev/null +++ b/.changeset/strange-rocks-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table. diff --git a/plugins/catalog-backend/migrations/20221109192547_search_add_original_value_column.js b/plugins/catalog-backend/migrations/20221109192547_search_add_original_value_column.js new file mode 100644 index 0000000000..f931b7ef89 --- /dev/null +++ b/plugins/catalog-backend/migrations/20221109192547_search_add_original_value_column.js @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('search', table => { + table + .string('original_value') + .nullable() + .comment('Holds the corresponding original case sensitive value'); + table.index(['original_value'], 'search_original_value_idx'); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('search', table => { + table.dropIndex([], 'search_original_value_idx'); + table.dropColumn('original_value') + }); +}; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index b9fe12be11..c23753b267 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -71,5 +71,6 @@ export type DbFinalEntitiesRow = { export type DbSearchRow = { entity_id: string; key: string; + original_value: string | null; value: string | null; }; diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 515a862515..2c75785dad 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -26,6 +26,7 @@ import { DbSearchRow, } from '../database/tables'; import { Stitcher } from '../stitching/Stitcher'; +import { buildEntitySearch } from '../stitching/buildEntitySearch'; import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; describe('DefaultEntitiesCatalog', () => { @@ -100,29 +101,14 @@ describe('DefaultEntitiesCatalog', () => { stitch_ticket: '', }); - await insertSearchRow(knex, id, null, entity); - } - - async function insertSearchRow( - knex: Knex, - id: string, - previousKey: string | null, - previousValue: Object, - ) { - return Promise.all( - Object.entries(previousValue).map(async ([key, value]) => { - const currentKey = `${previousKey ? `${previousKey}.` : ``}${key}`; - if (typeof value === 'object') { - await insertSearchRow(knex, id, currentKey, value); - } else { - await knex('search').insert({ - entity_id: id, - key: currentKey, - value: value, - }); - } - }), - ); + for (const row of buildEntitySearch(id, entity)) { + await knex('search').insert({ + entity_id: id, + key: row.key, + value: row.value, + original_value: row.original_value, + }); + } } afterEach(() => { @@ -792,8 +778,8 @@ describe('DefaultEntitiesCatalog', () => { facets: { 'metadata.tags': expect.arrayContaining([ { value: 'java', count: 2 }, - { value: 'rust', count: 1 }, { value: 'node', count: 1 }, + { value: 'rust', count: 1 }, ]), }, }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 81de6484c3..c7e41fd631 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -21,7 +21,6 @@ import { } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; -import lodash from 'lodash'; import { EntitiesBatchRequest, EntitiesBatchResponse, @@ -91,12 +90,13 @@ function addCondition( db: Knex, filter: EntitiesSearchFilter, negate: boolean = false, + entityIdField = 'entity_id', ) { // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. const matchQuery = db('search') - .select('entity_id') + .select(entityIdField) .where({ key: filter.key.toLowerCase() }) .andWhere(function keyFilter() { if (filter.values) { @@ -111,7 +111,7 @@ function addCondition( } } }); - queryBuilder.andWhere('entity_id', negate ? 'not in' : 'in', matchQuery); + queryBuilder.andWhere(entityIdField, negate ? 'not in' : 'in', matchQuery); } function isEntitiesSearchFilter( @@ -137,25 +137,30 @@ function parseFilter( query: Knex.QueryBuilder, db: Knex, negate: boolean = false, + entityIdField = 'entity_id', ): Knex.QueryBuilder { if (isEntitiesSearchFilter(filter)) { return query.andWhere(function filterFunction() { - addCondition(this, db, filter, negate); + addCondition(this, db, filter, negate, entityIdField); }); } if (isNegationEntityFilter(filter)) { - return parseFilter(filter.not, query, db, !negate); + return parseFilter(filter.not, query, db, !negate, entityIdField); } return query[negate ? 'andWhereNot' : 'andWhere'](function filterFunction() { if (isOrEntityFilter(filter)) { for (const subFilter of filter.anyOf ?? []) { - this.orWhere(subQuery => parseFilter(subFilter, subQuery, db)); + this.orWhere(subQuery => + parseFilter(subFilter, subQuery, db, false, entityIdField), + ); } } else { for (const subFilter of filter.allOf ?? []) { - this.andWhere(subQuery => parseFilter(subFilter, subQuery, db)); + this.andWhere(subQuery => + parseFilter(subFilter, subQuery, db, false, entityIdField), + ); } } }); @@ -190,7 +195,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } let rows = await entitiesQuery; - let pageInfo: DbPageInfo; if (limit === undefined || rows.length <= limit) { pageInfo = { hasNextPage: false }; @@ -392,47 +396,30 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async facets(request: EntityFacetsRequest): Promise { - const { entities } = await this.entities({ - filter: request.filter, - authorizationToken: request.authorizationToken, - }); - const facets: EntityFacetsResponse['facets'] = {}; + const db = this.database; for (const facet of request.facets) { - const values = entities - .map(entity => { - // TODO(freben): Generalize this code to handle any field that may - // have dots in its key? - if (facet.startsWith('metadata.annotations.')) { - return entity.metadata.annotations?.[ - facet.substring('metadata.annotations.'.length) - ]; - } else if (facet.startsWith('metadata.labels.')) { - return entity.metadata.labels?.[ - facet.substring('metadata.labels.'.length) - ]; - } - return lodash.get(entity, facet); + const dbQuery = db('search') + .join('final_entities', 'search.entity_id', 'final_entities.entity_id') + .where('search.key', facet.toLowerCase()) + .count('search.entity_id as count') + .select({ + value: 'search.original_value', }) - .flatMap(field => { - if (typeof field === 'string') { - return [field]; - } else if (Array.isArray(field)) { - return field.filter(i => typeof i === 'string'); - } - return []; - }) - .sort(); + .groupBy('search.original_value'); - const counts = lodash.countBy(values, lodash.identity); + if (request?.filter) { + parseFilter(request.filter, dbQuery, db, false, 'search.entity_id'); + } - facets[facet] = Object.entries(counts).map(([value, count]) => ({ - value, - count, + const result = await dbQuery; + + facets[facet] = result.map(data => ({ + value: data.value as string, + count: data.count as number, })); } - return { facets }; } } diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index 9accebf1fc..b42e68d105 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -115,13 +115,43 @@ describe('Stitcher', () => { const search = await db('search'); expect(search).toEqual( expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + { + entity_id: 'my-id', + key: 'relations.looksat', + original_value: 'k:ns/other', + value: 'k:ns/other', + }, + { + entity_id: 'my-id', + key: 'apiversion', + original_value: 'a', + value: 'a', + }, + { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, + { + entity_id: 'my-id', + key: 'metadata.namespace', + original_value: 'ns', + value: 'ns', + }, + { + entity_id: 'my-id', + key: 'metadata.uid', + original_value: 'my-id', + value: 'my-id', + }, + { + entity_id: 'my-id', + key: 'spec.k', + original_value: 'v', + value: 'v', + }, ]), ); @@ -179,14 +209,49 @@ describe('Stitcher', () => { expect(await db('search')).toEqual( expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + { + entity_id: 'my-id', + key: 'relations.looksat', + original_value: 'k:ns/other', + value: 'k:ns/other', + }, + { + entity_id: 'my-id', + key: 'relations.looksat', + original_value: 'k:ns/third', + value: 'k:ns/third', + }, + { + entity_id: 'my-id', + key: 'apiversion', + original_value: 'a', + value: 'a', + }, + { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, + { + entity_id: 'my-id', + key: 'metadata.namespace', + original_value: 'ns', + value: 'ns', + }, + { + entity_id: 'my-id', + key: 'metadata.uid', + original_value: 'my-id', + value: 'my-id', + }, + { + entity_id: 'my-id', + key: 'spec.k', + original_value: 'v', + value: 'v', + }, ]), ); }, diff --git a/plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts b/plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts index 7364486f06..384794d230 100644 --- a/plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts +++ b/plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts @@ -88,19 +88,26 @@ describe('buildEntitySearch', () => { ]; const output = mapToRows(input, 'eid'); expect(output).toEqual([ - { entity_id: 'eid', key: 'a', value: 'true' }, - { entity_id: 'eid', key: 'b', value: 'false' }, - { entity_id: 'eid', key: 'c', value: '7' }, - { entity_id: 'eid', key: 'd', value: 'string' }, - { entity_id: 'eid', key: 'e', value: null }, - { entity_id: 'eid', key: 'f', value: null }, + { entity_id: 'eid', key: 'a', original_value: 'true', value: 'true' }, + { entity_id: 'eid', key: 'b', original_value: 'false', value: 'false' }, + { entity_id: 'eid', key: 'c', original_value: '7', value: '7' }, + { + entity_id: 'eid', + key: 'd', + original_value: 'string', + value: 'string', + }, + { entity_id: 'eid', key: 'e', original_value: null, value: null }, + { entity_id: 'eid', key: 'f', original_value: null, value: null }, ]); }); - it('emits lowercase version of keys and values', () => { + it('emits lowercase version of keys and values and also keeps the original value', () => { const input = [{ key: 'fOo', value: 'BaR' }]; const output = mapToRows(input, 'eid'); - expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]); + expect(output).toEqual([ + { entity_id: 'eid', key: 'foo', original_value: 'BaR', value: 'bar' }, + ]); }); it('skips very large keys', () => { @@ -112,7 +119,9 @@ describe('buildEntitySearch', () => { it('replaces very large values with null', () => { const input = [{ key: 'foo', value: 'a'.repeat(10000) }]; const output = mapToRows(input, 'eid'); - expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: null }]); + expect(output).toEqual([ + { entity_id: 'eid', key: 'foo', original_value: null, value: null }, + ]); }); }); @@ -124,14 +133,35 @@ describe('buildEntitySearch', () => { metadata: { name: 'n' }, }; expect(buildEntitySearch('eid', input)).toEqual([ - { entity_id: 'eid', key: 'apiversion', value: 'a' }, - { entity_id: 'eid', key: 'kind', value: 'b' }, - { entity_id: 'eid', key: 'metadata.name', value: 'n' }, - { entity_id: 'eid', key: 'metadata.namespace', value: null }, - { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'apiversion', + original_value: 'a', + value: 'a', + }, + { entity_id: 'eid', key: 'kind', original_value: 'b', value: 'b' }, + { + entity_id: 'eid', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, { entity_id: 'eid', key: 'metadata.namespace', + original_value: null, + value: null, + }, + { + entity_id: 'eid', + key: 'metadata.uid', + original_value: null, + value: null, + }, + { + entity_id: 'eid', + key: 'metadata.namespace', + original_value: DEFAULT_NAMESPACE, value: DEFAULT_NAMESPACE, }, ]); @@ -154,18 +184,49 @@ describe('buildEntitySearch', () => { metadata: { name: 'n' }, }; expect(buildEntitySearch('eid', input)).toEqual([ - { entity_id: 'eid', key: 'apiversion', value: 'a' }, - { entity_id: 'eid', key: 'kind', value: 'b' }, - { entity_id: 'eid', key: 'metadata.name', value: 'n' }, - { entity_id: 'eid', key: 'metadata.namespace', value: null }, - { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'apiversion', + original_value: 'a', + value: 'a', + }, + { entity_id: 'eid', key: 'kind', original_value: 'b', value: 'b' }, + { + entity_id: 'eid', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, { entity_id: 'eid', key: 'metadata.namespace', + original_value: null, + value: null, + }, + { + entity_id: 'eid', + key: 'metadata.uid', + original_value: null, + value: null, + }, + { + entity_id: 'eid', + key: 'metadata.namespace', + original_value: DEFAULT_NAMESPACE, value: DEFAULT_NAMESPACE, }, - { entity_id: 'eid', key: 'relations.t1', value: 'k:ns/a' }, - { entity_id: 'eid', key: 'relations.t2', value: 'k:ns/b' }, + { + entity_id: 'eid', + key: 'relations.t1', + original_value: 'k:ns/a', + value: 'k:ns/a', + }, + { + entity_id: 'eid', + key: 'relations.t2', + original_value: 'k:ns/b', + value: 'k:ns/b', + }, ]); }); diff --git a/plugins/catalog-backend/src/stitching/buildEntitySearch.ts b/plugins/catalog-backend/src/stitching/buildEntitySearch.ts index 88a87df134..b97083f207 100644 --- a/plugins/catalog-backend/src/stitching/buildEntitySearch.ts +++ b/plugins/catalog-backend/src/stitching/buildEntitySearch.ts @@ -134,15 +134,30 @@ export function mapToRows(input: Kv[], entityId: string): DbSearchRow[] { for (const { key: rawKey, value: rawValue } of input) { const key = rawKey.toLocaleLowerCase('en-US'); if (rawValue === undefined || rawValue === null) { - result.push({ entity_id: entityId, key, value: null }); + result.push({ + entity_id: entityId, + key, + original_value: null, + value: null, + }); } else { const value = String(rawValue).toLocaleLowerCase('en-US'); if (key.length <= MAX_KEY_LENGTH) { - result.push({ - entity_id: entityId, - key, - value: value.length <= MAX_VALUE_LENGTH ? value : null, - }); + if (value.length <= MAX_VALUE_LENGTH) { + result.push({ + entity_id: entityId, + key, + original_value: String(rawValue), + value: value, + }); + } else { + result.push({ + entity_id: entityId, + key, + original_value: null, + value: null, + }); + } } } }