Changes in facets endpoint to use search table

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Added original value of entity to a search table

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Add original values while mapping to search table rows

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Modified facets query to use search table

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Added changeset

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Modified the related tests

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

prettier changes and more test changes

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

modified mapToRows function

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Created a new migrations file to hold the new changes

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

Prettier changes

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

changes in migrations file

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

addressed comments

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

lint changes and changes to insert rows in a better way for tests

Signed-off-by: Aishwarya Manicka Ravichandran <airavichandra@expediagroup.com>

.changeset/strange-rocks-hide.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table.

plugins/catalog-backend/migrations/20221109192547_search_add_original_value_column.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param { import("knex").Knex } knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('search', table => {
+    table
+      .string('original_value')
+      .nullable()
+      .comment('Holds the corresponding original case sensitive value');
+    table.index(['original_value'], 'search_original_value_idx');
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('search', table => {
+    table.dropIndex([], 'search_original_value_idx');
+    table.dropColumn('original_value')
+  });
+};

plugins/catalog-backend/src/database/tables.ts

@@ -71,5 +71,6 @@ export type DbFinalEntitiesRow = {
 export type DbSearchRow = {
   entity_id: string;
   key: string;
+  original_value: string | null;
   value: string | null;
 };

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts

@@ -26,6 +26,7 @@ import {
   DbSearchRow,
 } from '../database/tables';
 import { Stitcher } from '../stitching/Stitcher';
+import { buildEntitySearch } from '../stitching/buildEntitySearch';
 import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
 
 describe('DefaultEntitiesCatalog', () => {
@@ -100,29 +101,14 @@ describe('DefaultEntitiesCatalog', () => {
       stitch_ticket: '',
     });
 
-    await insertSearchRow(knex, id, null, entity);
-  }
-
-  async function insertSearchRow(
-    knex: Knex,
-    id: string,
-    previousKey: string | null,
-    previousValue: Object,
-  ) {
-    return Promise.all(
-      Object.entries(previousValue).map(async ([key, value]) => {
-        const currentKey = `${previousKey ? `${previousKey}.` : ``}${key}`;
-        if (typeof value === 'object') {
-          await insertSearchRow(knex, id, currentKey, value);
-        } else {
-          await knex<DbSearchRow>('search').insert({
-            entity_id: id,
-            key: currentKey,
-            value: value,
-          });
-        }
-      }),
-    );
+    for (const row of buildEntitySearch(id, entity)) {
+      await knex<DbSearchRow>('search').insert({
+        entity_id: id,
+        key: row.key,
+        value: row.value,
+        original_value: row.original_value,
+      });
+    }
   }
 
   afterEach(() => {
@@ -792,8 +778,8 @@ describe('DefaultEntitiesCatalog', () => {
           facets: {
             'metadata.tags': expect.arrayContaining([
               { value: 'java', count: 2 },
-              { value: 'rust', count: 1 },
               { value: 'node', count: 1 },
+              { value: 'rust', count: 1 },
             ]),
           },
         });

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -21,7 +21,6 @@ import {
 } from '@backstage/catalog-model';
 import { InputError, NotFoundError } from '@backstage/errors';
 import { Knex } from 'knex';
-import lodash from 'lodash';
 import {
   EntitiesBatchRequest,
   EntitiesBatchResponse,
@@ -91,12 +90,13 @@ function addCondition(
   db: Knex,
   filter: EntitiesSearchFilter,
   negate: boolean = false,
+  entityIdField = 'entity_id',
 ) {
   // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
   // make a lot of sense. However, it had abysmal performance on sqlite
   // when datasets grew large, so we're using IN instead.
   const matchQuery = db<DbSearchRow>('search')
-    .select('entity_id')
+    .select(entityIdField)
     .where({ key: filter.key.toLowerCase() })
     .andWhere(function keyFilter() {
       if (filter.values) {
@@ -111,7 +111,7 @@ function addCondition(
         }
       }
     });
-  queryBuilder.andWhere('entity_id', negate ? 'not in' : 'in', matchQuery);
+  queryBuilder.andWhere(entityIdField, negate ? 'not in' : 'in', matchQuery);
 }
 
 function isEntitiesSearchFilter(
@@ -137,25 +137,30 @@ function parseFilter(
   query: Knex.QueryBuilder,
   db: Knex,
   negate: boolean = false,
+  entityIdField = 'entity_id',
 ): Knex.QueryBuilder {
   if (isEntitiesSearchFilter(filter)) {
     return query.andWhere(function filterFunction() {
-      addCondition(this, db, filter, negate);
+      addCondition(this, db, filter, negate, entityIdField);
     });
   }
 
   if (isNegationEntityFilter(filter)) {
-    return parseFilter(filter.not, query, db, !negate);
+    return parseFilter(filter.not, query, db, !negate, entityIdField);
   }
 
   return query[negate ? 'andWhereNot' : 'andWhere'](function filterFunction() {
     if (isOrEntityFilter(filter)) {
       for (const subFilter of filter.anyOf ?? []) {
-        this.orWhere(subQuery => parseFilter(subFilter, subQuery, db));
+        this.orWhere(subQuery =>
+          parseFilter(subFilter, subQuery, db, false, entityIdField),
+        );
       }
     } else {
       for (const subFilter of filter.allOf ?? []) {
-        this.andWhere(subQuery => parseFilter(subFilter, subQuery, db));
+        this.andWhere(subQuery =>
+          parseFilter(subFilter, subQuery, db, false, entityIdField),
+        );
       }
     }
   });
@@ -190,7 +195,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     }
 
     let rows = await entitiesQuery;
-
     let pageInfo: DbPageInfo;
     if (limit === undefined || rows.length <= limit) {
       pageInfo = { hasNextPage: false };
@@ -392,47 +396,30 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
   }
 
   async facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse> {
-    const { entities } = await this.entities({
-      filter: request.filter,
-      authorizationToken: request.authorizationToken,
-    });
-
     const facets: EntityFacetsResponse['facets'] = {};
+    const db = this.database;
 
     for (const facet of request.facets) {
-      const values = entities
-        .map(entity => {
-          // TODO(freben): Generalize this code to handle any field that may
-          // have dots in its key?
-          if (facet.startsWith('metadata.annotations.')) {
-            return entity.metadata.annotations?.[
-              facet.substring('metadata.annotations.'.length)
-            ];
-          } else if (facet.startsWith('metadata.labels.')) {
-            return entity.metadata.labels?.[
-              facet.substring('metadata.labels.'.length)
-            ];
-          }
-          return lodash.get(entity, facet);
+      const dbQuery = db<DbSearchRow>('search')
+        .join('final_entities', 'search.entity_id', 'final_entities.entity_id')
+        .where('search.key', facet.toLowerCase())
+        .count('search.entity_id as count')
+        .select({
+          value: 'search.original_value',
         })
-        .flatMap(field => {
-          if (typeof field === 'string') {
-            return [field];
-          } else if (Array.isArray(field)) {
-            return field.filter(i => typeof i === 'string');
-          }
-          return [];
-        })
-        .sort();
+        .groupBy('search.original_value');
 
-      const counts = lodash.countBy(values, lodash.identity);
+      if (request?.filter) {
+        parseFilter(request.filter, dbQuery, db, false, 'search.entity_id');
+      }
 
-      facets[facet] = Object.entries(counts).map(([value, count]) => ({
-        value,
-        count,
+      const result = await dbQuery;
+
+      facets[facet] = result.map(data => ({
+        value: data.value as string,
+        count: data.count as number,
       }));
     }
-
     return { facets };
   }
 }

plugins/catalog-backend/src/stitching/Stitcher.test.ts

@@ -115,13 +115,43 @@ describe('Stitcher', () => {
       const search = await db<DbSearchRow>('search');
       expect(search).toEqual(
         expect.arrayContaining([
-          { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
-          { entity_id: 'my-id', key: 'apiversion', value: 'a' },
-          { entity_id: 'my-id', key: 'kind', value: 'k' },
-          { entity_id: 'my-id', key: 'metadata.name', value: 'n' },
-          { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
-          { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
-          { entity_id: 'my-id', key: 'spec.k', value: 'v' },
+          {
+            entity_id: 'my-id',
+            key: 'relations.looksat',
+            original_value: 'k:ns/other',
+            value: 'k:ns/other',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'apiversion',
+            original_value: 'a',
+            value: 'a',
+          },
+          { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.name',
+            original_value: 'n',
+            value: 'n',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.namespace',
+            original_value: 'ns',
+            value: 'ns',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.uid',
+            original_value: 'my-id',
+            value: 'my-id',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'spec.k',
+            original_value: 'v',
+            value: 'v',
+          },
         ]),
       );
 
@@ -179,14 +209,49 @@ describe('Stitcher', () => {
 
       expect(await db<DbSearchRow>('search')).toEqual(
         expect.arrayContaining([
-          { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
-          { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' },
-          { entity_id: 'my-id', key: 'apiversion', value: 'a' },
-          { entity_id: 'my-id', key: 'kind', value: 'k' },
-          { entity_id: 'my-id', key: 'metadata.name', value: 'n' },
-          { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
-          { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
-          { entity_id: 'my-id', key: 'spec.k', value: 'v' },
+          {
+            entity_id: 'my-id',
+            key: 'relations.looksat',
+            original_value: 'k:ns/other',
+            value: 'k:ns/other',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'relations.looksat',
+            original_value: 'k:ns/third',
+            value: 'k:ns/third',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'apiversion',
+            original_value: 'a',
+            value: 'a',
+          },
+          { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.name',
+            original_value: 'n',
+            value: 'n',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.namespace',
+            original_value: 'ns',
+            value: 'ns',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'metadata.uid',
+            original_value: 'my-id',
+            value: 'my-id',
+          },
+          {
+            entity_id: 'my-id',
+            key: 'spec.k',
+            original_value: 'v',
+            value: 'v',
+          },
         ]),
       );
     },

plugins/catalog-backend/src/stitching/buildEntitySearch.test.ts

@@ -88,19 +88,26 @@ describe('buildEntitySearch', () => {
       ];
       const output = mapToRows(input, 'eid');
       expect(output).toEqual([
-        { entity_id: 'eid', key: 'a', value: 'true' },
-        { entity_id: 'eid', key: 'b', value: 'false' },
-        { entity_id: 'eid', key: 'c', value: '7' },
-        { entity_id: 'eid', key: 'd', value: 'string' },
-        { entity_id: 'eid', key: 'e', value: null },
-        { entity_id: 'eid', key: 'f', value: null },
+        { entity_id: 'eid', key: 'a', original_value: 'true', value: 'true' },
+        { entity_id: 'eid', key: 'b', original_value: 'false', value: 'false' },
+        { entity_id: 'eid', key: 'c', original_value: '7', value: '7' },
+        {
+          entity_id: 'eid',
+          key: 'd',
+          original_value: 'string',
+          value: 'string',
+        },
+        { entity_id: 'eid', key: 'e', original_value: null, value: null },
+        { entity_id: 'eid', key: 'f', original_value: null, value: null },
       ]);
     });
 
-    it('emits lowercase version of keys and values', () => {
+    it('emits lowercase version of keys and values and also keeps the original value', () => {
       const input = [{ key: 'fOo', value: 'BaR' }];
       const output = mapToRows(input, 'eid');
-      expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'foo', original_value: 'BaR', value: 'bar' },
+      ]);
     });
 
     it('skips very large keys', () => {
@@ -112,7 +119,9 @@ describe('buildEntitySearch', () => {
     it('replaces very large values with null', () => {
       const input = [{ key: 'foo', value: 'a'.repeat(10000) }];
       const output = mapToRows(input, 'eid');
-      expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: null }]);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'foo', original_value: null, value: null },
+      ]);
     });
   });
 
@@ -124,14 +133,35 @@ describe('buildEntitySearch', () => {
         metadata: { name: 'n' },
       };
       expect(buildEntitySearch('eid', input)).toEqual([
-        { entity_id: 'eid', key: 'apiversion', value: 'a' },
-        { entity_id: 'eid', key: 'kind', value: 'b' },
-        { entity_id: 'eid', key: 'metadata.name', value: 'n' },
-        { entity_id: 'eid', key: 'metadata.namespace', value: null },
-        { entity_id: 'eid', key: 'metadata.uid', value: null },
+        {
+          entity_id: 'eid',
+          key: 'apiversion',
+          original_value: 'a',
+          value: 'a',
+        },
+        { entity_id: 'eid', key: 'kind', original_value: 'b', value: 'b' },
+        {
+          entity_id: 'eid',
+          key: 'metadata.name',
+          original_value: 'n',
+          value: 'n',
+        },
         {
           entity_id: 'eid',
           key: 'metadata.namespace',
+          original_value: null,
+          value: null,
+        },
+        {
+          entity_id: 'eid',
+          key: 'metadata.uid',
+          original_value: null,
+          value: null,
+        },
+        {
+          entity_id: 'eid',
+          key: 'metadata.namespace',
+          original_value: DEFAULT_NAMESPACE,
           value: DEFAULT_NAMESPACE,
         },
       ]);
@@ -154,18 +184,49 @@ describe('buildEntitySearch', () => {
         metadata: { name: 'n' },
       };
       expect(buildEntitySearch('eid', input)).toEqual([
-        { entity_id: 'eid', key: 'apiversion', value: 'a' },
-        { entity_id: 'eid', key: 'kind', value: 'b' },
-        { entity_id: 'eid', key: 'metadata.name', value: 'n' },
-        { entity_id: 'eid', key: 'metadata.namespace', value: null },
-        { entity_id: 'eid', key: 'metadata.uid', value: null },
+        {
+          entity_id: 'eid',
+          key: 'apiversion',
+          original_value: 'a',
+          value: 'a',
+        },
+        { entity_id: 'eid', key: 'kind', original_value: 'b', value: 'b' },
+        {
+          entity_id: 'eid',
+          key: 'metadata.name',
+          original_value: 'n',
+          value: 'n',
+        },
         {
           entity_id: 'eid',
           key: 'metadata.namespace',
+          original_value: null,
+          value: null,
+        },
+        {
+          entity_id: 'eid',
+          key: 'metadata.uid',
+          original_value: null,
+          value: null,
+        },
+        {
+          entity_id: 'eid',
+          key: 'metadata.namespace',
+          original_value: DEFAULT_NAMESPACE,
           value: DEFAULT_NAMESPACE,
         },
-        { entity_id: 'eid', key: 'relations.t1', value: 'k:ns/a' },
-        { entity_id: 'eid', key: 'relations.t2', value: 'k:ns/b' },
+        {
+          entity_id: 'eid',
+          key: 'relations.t1',
+          original_value: 'k:ns/a',
+          value: 'k:ns/a',
+        },
+        {
+          entity_id: 'eid',
+          key: 'relations.t2',
+          original_value: 'k:ns/b',
+          value: 'k:ns/b',
+        },
       ]);
     });
 

plugins/catalog-backend/src/stitching/buildEntitySearch.ts

@@ -134,15 +134,30 @@ export function mapToRows(input: Kv[], entityId: string): DbSearchRow[] {
   for (const { key: rawKey, value: rawValue } of input) {
     const key = rawKey.toLocaleLowerCase('en-US');
     if (rawValue === undefined || rawValue === null) {
-      result.push({ entity_id: entityId, key, value: null });
+      result.push({
+        entity_id: entityId,
+        key,
+        original_value: null,
+        value: null,
+      });
     } else {
       const value = String(rawValue).toLocaleLowerCase('en-US');
       if (key.length <= MAX_KEY_LENGTH) {
-        result.push({
-          entity_id: entityId,
-          key,
-          value: value.length <= MAX_VALUE_LENGTH ? value : null,
-        });
+        if (value.length <= MAX_VALUE_LENGTH) {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: String(rawValue),
+            value: value,
+          });
+        } else {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: null,
+            value: null,
+          });
+        }
       }
     }
   }