Merge pull request #12246 from backstage/vinzscam/catalog-backend-paginated-entities

Catalog: paginated entities endpoint

plugins/catalog-backend/src/catalog/types.ts

@@ -174,6 +174,13 @@ export interface EntitiesCatalog {
    */
   entitiesBatch(request: EntitiesBatchRequest): Promise<EntitiesBatchResponse>;
 
+  /**
+   * Fetch entities and scroll back and forth between entities.
+   *
+   * @param request
+   */
+  queryEntities(request: QueryEntitiesRequest): Promise<QueryEntitiesResponse>;
+
   /**
    * Removes a single entity.
    *
@@ -202,3 +209,103 @@ export interface EntitiesCatalog {
    */
   facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse>;
 }
+
+/**
+ * The request shape for {@link EntitiesCatalog.queryEntities}.
+ */
+export type QueryEntitiesRequest =
+  | QueryEntitiesInitialRequest
+  | QueryEntitiesCursorRequest;
+
+/**
+ * The initial request for {@link EntitiesCatalog.queryEntities}.
+ * The request take immutable properties that are going to be bound
+ * for the current and the next pagination requests.
+ */
+export interface QueryEntitiesInitialRequest {
+  authorizationToken?: string;
+  fields?: (entity: Entity) => Entity;
+  limit?: number;
+  filter?: EntityFilter;
+  orderFields?: EntityOrder[];
+  fullTextFilter?: {
+    term: string;
+    fields?: string[];
+  };
+}
+
+/**
+ * Request for {@link EntitiesCatalog.queryEntities} used to
+ * move forward or backward on the data.
+ */
+export interface QueryEntitiesCursorRequest {
+  authorizationToken?: string;
+  fields?: (entity: Entity) => Entity;
+  limit?: number;
+  cursor: Cursor;
+}
+
+/**
+ * The response shape for {@link EntitiesCatalog.queryEntities}.
+ */
+export interface QueryEntitiesResponse {
+  /**
+   * The entities for the current pagination request
+   */
+  items: Entity[];
+
+  pageInfo: {
+    /**
+     * The cursor of the next pagination request.
+     */
+    nextCursor?: Cursor;
+    /**
+     * The cursor of the previous pagination request.
+     */
+    prevCursor?: Cursor;
+  };
+  /**
+   * the total number of entities matching the current filters.
+   */
+  totalItems: number;
+}
+
+/**
+ * The Cursor used internally by the catalog.
+ */
+export type Cursor = {
+  /**
+   * An array of fields used for sorting the data.
+   * For example, [ { field: 'metadata.name', order: 'asc' } ]
+   */
+  orderFields: EntityOrder[];
+  /**
+   * The values of the fields of a specific item used for paginating the data.
+   */
+  orderFieldValues: Array<string | null>;
+  /**
+   * A filter to be applied to the full list of entities.
+   */
+  filter?: EntityFilter;
+  /**
+   * true if the cursor is a previous cursor.
+   */
+  isPrevious: boolean;
+  /**
+   * Used for performing full text filtering on the given fields.
+   */
+  fullTextFilter?: {
+    term: string;
+    fields?: string[];
+  };
+  /**
+   * The value of the fields of the first returned item used for paginating the data.
+   * The catalog uses this field internally to understand if the beginning
+   * of the list has been reached when performing cursor based pagination.
+   */
+  firstSortFieldValues?: Array<string | null>;
+  /**
+   * The number of items that match the provided filters
+   */
+  totalItems?: number;
+};

plugins/catalog-backend/src/integration.test.ts

@@ -268,7 +268,11 @@ class TestHarness {
       legacySingleProcessorValidation: false,
     });
     const stitcher = new Stitcher(db, logger);
-    const catalog = new DefaultEntitiesCatalog(db, stitcher);
+    const catalog = new DefaultEntitiesCatalog({
+      database: db,
+      logger,
+      stitcher,
+    });
     const proxyProgressTracker = new ProxyProgressTracker(
       new NoopProgressTracker(),
     );

plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts

@@ -20,6 +20,8 @@ import { createConditionTransformer } from '@backstage/plugin-permission-node';
 import { isEntityKind } from '../permissions/rules/isEntityKind';
 import { CatalogPermissionRule } from '../permissions/rules';
 import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog';
+import { Cursor, EntityFilter, QueryEntitiesResponse } from '../catalog/types';
+import { Entity } from '@backstage/catalog-model';
 
 describe('AuthorizedEntitiesCatalog', () => {
   const fakeCatalog = {
@@ -30,6 +32,7 @@ describe('AuthorizedEntitiesCatalog', () => {
     facets: jest.fn(),
     refresh: jest.fn(),
     listAncestors: jest.fn(),
+    queryEntities: jest.fn(),
   };
   const fakePermissionApi = {
     authorize: jest.fn(),
@@ -156,6 +159,153 @@ describe('AuthorizedEntitiesCatalog', () => {
     });
   });
 
+  describe('queryEntities', () => {
+    it('returns empty response on DENY', async () => {
+      fakePermissionApi.authorizeConditional.mockResolvedValue([
+        { result: AuthorizeResult.DENY },
+      ]);
+      const catalog = createCatalog();
+
+      await expect(
+        catalog.queryEntities({
+          authorizationToken: 'abcd',
+          filter: { key: 'kind', values: ['b'] },
+        }),
+      ).resolves.toEqual({
+        items: [],
+        pageInfo: {},
+        totalItems: 0,
+      });
+
+      expect(fakeCatalog.queryEntities).not.toHaveBeenCalled();
+    });
+
+    it('calls underlying catalog method on ALLOW', async () => {
+      fakePermissionApi.authorizeConditional.mockResolvedValue([
+        { result: AuthorizeResult.ALLOW },
+      ]);
+      const catalog = createCatalog();
+
+      await catalog.queryEntities({
+        authorizationToken: 'abcd',
+        filter: { key: 'kind', values: ['b'] },
+      });
+
+      expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({
+        authorizationToken: 'abcd',
+        filter: { key: 'kind', values: ['b'] },
+      });
+    });
+
+    it('calls underlying catalog method with correct filter on CONDITIONAL', async () => {
+      fakePermissionApi.authorizeConditional.mockResolvedValue([
+        {
+          result: AuthorizeResult.CONDITIONAL,
+          conditions: {
+            rule: 'IS_ENTITY_KIND',
+            params: { kinds: ['b'] },
+          },
+        },
+      ]);
+
+      const requestFilter: EntityFilter = { key: 'name', values: ['name'] };
+
+      const entities = [
+        {
+          kind: 'component',
+          namespace: 'default',
+          name: 'a',
+        } as unknown as Entity,
+        {
+          kind: 'component',
+          namespace: 'default',
+          name: 'b1',
+        } as unknown as Entity,
+      ];
+
+      fakeCatalog.queryEntities.mockResolvedValue({
+        items: entities,
+        pageInfo: {
+          nextCursor: {
+            isPrevious: false,
+            orderFieldValues: ['xxx', null],
+            filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] },
+          },
+          prevCursor: {
+            isPrevious: true,
+            orderFieldValues: ['a', null],
+            filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] },
+          },
+        },
+        totalItems: 4,
+      } as QueryEntitiesResponse);
+      const catalog = createCatalog(isEntityKind);
+
+      let response = await catalog.queryEntities({
+        authorizationToken: 'abcd',
+        filter: { key: 'name', values: ['name'] },
+      });
+
+      expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({
+        authorizationToken: 'abcd',
+        filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] },
+      });
+
+      expect(response).toEqual({
+        items: entities,
+        totalItems: 4,
+        pageInfo: {
+          nextCursor: {
+            isPrevious: false,
+            filter: requestFilter,
+            orderFieldValues: ['xxx', null],
+          },
+          prevCursor: {
+            isPrevious: true,
+            filter: requestFilter,
+            orderFieldValues: ['a', null],
+          },
+        },
+      });
+
+      const cursor: Cursor = {
+        filter: requestFilter,
+        orderFields: [{ field: 'name', order: 'asc' }],
+        isPrevious: false,
+        orderFieldValues: ['a', null],
+      };
+      response = await catalog.queryEntities({
+        authorizationToken: 'abcd',
+        cursor,
+      });
+
+      expect(fakeCatalog.queryEntities).toHaveBeenNthCalledWith(2, {
+        authorizationToken: 'abcd',
+        cursor: {
+          ...cursor,
+          filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] },
+        },
+      });
+
+      expect(response).toEqual({
+        items: entities,
+        totalItems: 4,
+        pageInfo: {
+          nextCursor: {
+            isPrevious: false,
+            filter: requestFilter,
+            orderFieldValues: ['xxx', null],
+          },
+          prevCursor: {
+            isPrevious: true,
+            filter: requestFilter,
+            orderFieldValues: ['a', null],
+          },
+        },
+      });
+    });
+  });
+
   describe('removeEntityByUid', () => {
     it('throws error on DENY', async () => {
       fakeCatalog.entities.mockResolvedValue({

plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts

@@ -26,6 +26,7 @@ import {
 } from '@backstage/plugin-permission-common';
 import { ConditionTransformer } from '@backstage/plugin-permission-node';
 import {
+  Cursor,
   EntitiesBatchRequest,
   EntitiesBatchResponse,
   EntitiesCatalog,
@@ -35,8 +36,11 @@ import {
   EntityFacetsRequest,
   EntityFacetsResponse,
   EntityFilter,
+  QueryEntitiesRequest,
+  QueryEntitiesResponse,
 } from '../catalog/types';
 import { basicEntityFilter } from './request/basicEntityFilter';
+import { isQueryEntitiesCursorRequest } from './util';
 
 export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
   constructor(
@@ -106,6 +110,80 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
     return this.entitiesCatalog.entitiesBatch(request);
   }
 
+  async queryEntities(
+    request: QueryEntitiesRequest,
+  ): Promise<QueryEntitiesResponse> {
+    const authorizeDecision = (
+      await this.permissionApi.authorizeConditional(
+        [{ permission: catalogEntityReadPermission }],
+        { token: request.authorizationToken },
+      )
+    )[0];
+
+    if (authorizeDecision.result === AuthorizeResult.DENY) {
+      return {
+        items: [],
+        pageInfo: {},
+        totalItems: 0,
+      };
+    }
+
+    if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) {
+      const permissionFilter: EntityFilter = this.transformConditions(
+        authorizeDecision.conditions,
+      );
+
+      let permissionedRequest: QueryEntitiesRequest;
+      let requestFilter: EntityFilter | undefined;
+
+      if (isQueryEntitiesCursorRequest(request)) {
+        requestFilter = request.cursor.filter;
+
+        permissionedRequest = {
+          ...request,
+          cursor: {
+            ...request.cursor,
+            filter: request.cursor.filter
+              ? { allOf: [permissionFilter, request.cursor.filter] }
+              : permissionFilter,
+          },
+        };
+      } else {
+        permissionedRequest = {
+          ...request,
+          filter: request.filter
+            ? { allOf: [permissionFilter, request.filter] }
+            : permissionFilter,
+        };
+        requestFilter = request.filter;
+      }
+
+      const response = await this.entitiesCatalog.queryEntities(
+        permissionedRequest,
+      );
+
+      const prevCursor: Cursor | undefined = response.pageInfo.prevCursor && {
+        ...response.pageInfo.prevCursor,
+        filter: requestFilter,
+      };
+
+      const nextCursor: Cursor | undefined = response.pageInfo.nextCursor && {
+        ...response.pageInfo.nextCursor,
+        filter: requestFilter,
+      };
+
+      return {
+        ...response,
+        pageInfo: {
+          prevCursor,
+          nextCursor,
+        },
+      };
+    }
+
+    return this.entitiesCatalog.queryEntities(request);
+  }
+
   async removeEntityByUid(
     uid: string,
     options?: { authorizationToken?: string },

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -453,10 +453,11 @@ export class CatalogBuilder {
       legacySingleProcessorValidation: this.legacySingleProcessorValidation,
     });
     const stitcher = new Stitcher(dbClient, logger);
-    const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog(
-      dbClient,
+    const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog({
+      database: dbClient,
+      logger,
       stitcher,
-    );
+    });
 
     let permissionEvaluator: PermissionEvaluator;
     if ('authorizeConditional' in permissions) {

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -20,11 +20,14 @@ import {
   stringifyEntityRef,
 } from '@backstage/catalog-model';
 import { InputError, NotFoundError } from '@backstage/errors';
-import lodash from 'lodash';
 import { Knex } from 'knex';
+import { isEqual, chunk as lodashChunk } from 'lodash';
+import { Logger } from 'winston';
+import { z } from 'zod';
 import {
   EntitiesBatchRequest,
   EntitiesBatchResponse,
+  Cursor,
   EntitiesCatalog,
   EntitiesRequest,
   EntitiesResponse,
@@ -34,6 +37,9 @@ import {
   EntityFacetsResponse,
   EntityFilter,
   EntityPagination,
+  QueryEntitiesRequest,
+  QueryEntitiesResponse,
+  EntityOrder,
 } from '../catalog/types';
 import {
   DbFinalEntitiesRow,
@@ -43,8 +49,21 @@ import {
   DbRelationsRow,
   DbSearchRow,
 } from '../database/tables';
+
 import { Stitcher } from '../stitching/Stitcher';
 
+import {
+  isQueryEntitiesCursorRequest,
+  isQueryEntitiesInitialRequest,
+} from './util';
+
+const defaultSortField: EntityOrder = {
+  field: 'metadata.uid',
+  order: 'asc',
+};
+
+const DEFAULT_LIMIT = 20;
+
 function parsePagination(input?: EntityPagination): {
   limit?: number;
   offset?: number;
@@ -168,10 +187,15 @@ function parseFilter(
 }
 
 export class DefaultEntitiesCatalog implements EntitiesCatalog {
-  constructor(
-    private readonly database: Knex,
-    private readonly stitcher: Stitcher,
-  ) {}
+  private readonly database: Knex;
+  private readonly logger: Logger;
+  private readonly stitcher: Stitcher;
+
+  constructor(options: { database: Knex; logger: Logger; stitcher: Stitcher }) {
+    this.database = options.database;
+    this.logger = options.logger;
+    this.stitcher = options.stitcher;
+  }
 
   async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
     const db = this.database;
@@ -283,7 +307,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
   ): Promise<EntitiesBatchResponse> {
     const lookup = new Map<string, Entity>();
 
-    for (const chunk of lodash.chunk(request.entityRefs, 200)) {
+    for (const chunk of lodashChunk(request.entityRefs, 200)) {
       let query = this.database<DbFinalEntitiesRow>('final_entities')
         .innerJoin<DbRefreshStateRow>('refresh_state', {
           'refresh_state.entity_id': 'final_entities.entity_id',
@@ -310,6 +334,161 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     return { items };
   }
 
+  async queryEntities(
+    request: QueryEntitiesRequest,
+  ): Promise<QueryEntitiesResponse> {
+    const db = this.database;
+
+    const limit = request.limit ?? DEFAULT_LIMIT;
+
+    const cursor: Omit<Cursor, 'orderFieldValues'> & {
+      orderFieldValues?: (string | null)[];
+    } = {
+      orderFields: [defaultSortField],
+      isPrevious: false,
+      ...parseCursorFromRequest(request),
+    };
+
+    const isFetchingBackwards = cursor.isPrevious;
+
+    if (cursor.orderFields.length > 1) {
+      this.logger.warn(`Only one sort field is supported, ignoring the rest`);
+    }
+
+    const sortField: EntityOrder = {
+      ...defaultSortField,
+      ...cursor.orderFields[0],
+    };
+
+    const [prevItemOrderFieldValue, prevItemUid] =
+      cursor.orderFieldValues || [];
+
+    const dbQuery = db('search')
+      .join('final_entities', 'search.entity_id', 'final_entities.entity_id')
+      .where('search.key', sortField.field);
+
+    if (cursor.filter) {
+      parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id');
+    }
+
+    const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();
+    if (normalizedFullTextFilterTerm) {
+      dbQuery.andWhereRaw(
+        'value like ?',
+        `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,
+      );
+    }
+
+    const countQuery = dbQuery.clone();
+
+    const isOrderingDescending = sortField.order === 'desc';
+
+    if (prevItemOrderFieldValue) {
+      dbQuery.andWhere(
+        'value',
+        isFetchingBackwards !== isOrderingDescending ? '<' : '>',
+        prevItemOrderFieldValue,
+      );
+      dbQuery.orWhere(function nested() {
+        this.where('value', '=', prevItemOrderFieldValue).andWhere(
+          'search.entity_id',
+          isFetchingBackwards !== isOrderingDescending ? '<' : '>',
+          prevItemUid,
+        );
+      });
+    }
+
+    dbQuery
+      .orderBy([
+        {
+          column: 'value',
+          order: isFetchingBackwards
+            ? invertOrder(sortField.order)
+            : sortField.order,
+        },
+        {
+          column: 'search.entity_id',
+          order: isFetchingBackwards
+            ? invertOrder(sortField.order)
+            : sortField.order,
+        },
+      ])
+      // fetch an extra item to check if there are more items.
+      .limit(isFetchingBackwards ? limit : limit + 1);
+
+    countQuery.count('search.entity_id', { as: 'count' });
+
+    const [rows, [{ count }]] = await Promise.all([
+      dbQuery,
+      // for performance reasons we invoke the countQuery
+      // only on the first request.
+      // The result is then embedded into the cursor
+      // for subsequent requests.
+      typeof cursor.totalItems === 'undefined'
+        ? countQuery
+        : [{ count: cursor.totalItems }],
+    ]);
+
+    const totalItems = Number(count);
+
+    if (isFetchingBackwards) {
+      rows.reverse();
+    }
+    const hasMoreResults =
+      limit > 0 && (isFetchingBackwards || rows.length > limit);
+
+    // discard the extra item only when fetching forward.
+    if (rows.length > limit) {
+      rows.length -= 1;
+    }
+
+    const isInitialRequest = cursor.firstSortFieldValues === undefined;
+
+    const firstRow = rows[0];
+    const lastRow = rows[rows.length - 1];
+
+    const firstSortFieldValues = cursor.firstSortFieldValues || [
+      firstRow?.value,
+      firstRow?.entity_id,
+    ];
+
+    const nextCursor: Cursor | undefined = hasMoreResults
+      ? {
+          ...cursor,
+          orderFieldValues: sortFieldsFromRow(lastRow),
+          firstSortFieldValues,
+          isPrevious: false,
+          totalItems,
+        }
+      : undefined;
+
+    const prevCursor: Cursor | undefined =
+      !isInitialRequest &&
+      rows.length > 0 &&
+      !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues)
+        ? {
+            ...cursor,
+            orderFieldValues: sortFieldsFromRow(firstRow),
+            firstSortFieldValues: cursor.firstSortFieldValues,
+            isPrevious: true,
+            totalItems,
+          }
+        : undefined;
+
+    const items = rows
+      .map(e => JSON.parse(e.final_entity!))
+      .map(e => (request.fields ? request.fields(e) : e));
+
+    return {
+      items,
+      pageInfo: {
+        ...(!!prevCursor && { prevCursor }),
+        ...(!!nextCursor && { nextCursor }),
+      },
+      totalItems,
+    };
+  }
+
   async removeEntityByUid(uid: string): Promise<void> {
     const dbConfig = this.database.client.config;
 
@@ -490,3 +669,51 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     return { facets };
   }
 }
+
+const entityFilterParser: z.ZodSchema<EntityFilter> = z.lazy(() =>
+  z
+    .object({
+      key: z.string(),
+      values: z.array(z.string()).optional(),
+    })
+    .or(z.object({ not: entityFilterParser }))
+    .or(z.object({ anyOf: z.array(entityFilterParser) }))
+    .or(z.object({ allOf: z.array(entityFilterParser) })),
+);
+
+export const cursorParser: z.ZodSchema<Cursor> = z.object({
+  orderFields: z.array(
+    z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }),
+  ),
+  orderFieldValues: z.array(z.string().or(z.null())),
+  filter: entityFilterParser.optional(),
+  isPrevious: z.boolean(),
+  query: z.string().optional(),
+  firstSortFieldValues: z.array(z.string().or(z.null())).optional(),
+  totalItems: z.number().optional(),
+});
+
+function parseCursorFromRequest(
+  request?: QueryEntitiesRequest,
+): Partial<Cursor> {
+  if (isQueryEntitiesInitialRequest(request)) {
+    const {
+      filter,
+      orderFields: sortFields = [defaultSortField],
+      fullTextFilter,
+    } = request;
+    return { filter, orderFields: sortFields, fullTextFilter };
+  }
+  if (isQueryEntitiesCursorRequest(request)) {
+    return request.cursor;
+  }
+  return {};
+}
+
+function invertOrder(order: EntityOrder['order']) {
+  return order === 'asc' ? 'desc' : 'asc';
+}
+
+function sortFieldsFromRow(row: DbSearchRow) {
+  return [row.value, row.entity_id];
+}

plugins/catalog-backend/src/service/createRouter.test.ts

@@ -25,7 +25,7 @@ import {
 } from '@backstage/catalog-model';
 import express from 'express';
 import request from 'supertest';
-import { EntitiesCatalog } from '../catalog/types';
+import { Cursor, EntitiesCatalog } from '../catalog/types';
 import { LocationInput, LocationService, RefreshService } from './types';
 import { basicEntityFilter } from './request';
 import { createRouter } from './createRouter';
@@ -37,6 +37,7 @@ import {
 import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common';
 import { CatalogProcessingOrchestrator } from '../processing/types';
 import { z } from 'zod';
+import { encodeCursor } from './util';
 
 describe('createRouter readonly disabled', () => {
   let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
@@ -52,6 +53,7 @@ describe('createRouter readonly disabled', () => {
       removeEntityByUid: jest.fn(),
       entityAncestry: jest.fn(),
       facets: jest.fn(),
+      queryEntities: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),
@@ -135,6 +137,118 @@ describe('createRouter readonly disabled', () => {
     });
   });
 
+  describe('GET /entities/by-query', () => {
+    it('happy path: lists entities', async () => {
+      const items: Entity[] = [
+        { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
+      ];
+
+      entitiesCatalog.queryEntities.mockResolvedValueOnce({
+        items,
+        totalItems: 100,
+        pageInfo: { nextCursor: mockCursor() },
+      });
+
+      const response = await request(app).get('/entities/by-query');
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual({
+        items,
+        totalItems: 100,
+        pageInfo: {
+          nextCursor: expect.any(String),
+        },
+      });
+    });
+
+    it('parses initial request', async () => {
+      entitiesCatalog.queryEntities.mockResolvedValueOnce({
+        items: [],
+        pageInfo: {},
+        totalItems: 0,
+      });
+      const response = await request(app).get(
+        '/entities/by-query?filter=a=1,a=2,b=3&filter=c=4&orderField=metadata.name,asc&orderField=metadata.uid,desc',
+      );
+
+      expect(response.status).toEqual(200);
+      expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({
+        filter: {
+          anyOf: [
+            {
+              allOf: [
+                { key: 'a', values: ['1', '2'] },
+                { key: 'b', values: ['3'] },
+              ],
+            },
+            { allOf: [{ key: 'c', values: ['4'] }] },
+          ],
+        },
+        orderFields: [
+          { field: 'metadata.name', order: 'asc' },
+          { field: 'metadata.uid', order: 'desc' },
+        ],
+        fullTextFilter: {
+          fields: undefined,
+          term: '',
+        },
+      });
+    });
+
+    it('parses cursor request', async () => {
+      const items: Entity[] = [
+        { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
+      ];
+
+      entitiesCatalog.queryEntities.mockResolvedValueOnce({
+        items,
+        totalItems: 100,
+        pageInfo: { nextCursor: mockCursor() },
+      });
+
+      const cursor = mockCursor({ totalItems: 100, isPrevious: false });
+
+      const response = await request(app).get(
+        `/entities/by-query?cursor=${encodeCursor(cursor)}`,
+      );
+      expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({
+        cursor,
+      });
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual({
+        items,
+        totalItems: 100,
+        pageInfo: { nextCursor: expect.any(String) },
+      });
+    });
+
+    it('should throw in case of malformed cursor', async () => {
+      const items: Entity[] = [
+        { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
+      ];
+
+      entitiesCatalog.queryEntities.mockResolvedValueOnce({
+        items,
+        totalItems: 100,
+        pageInfo: { nextCursor: mockCursor() },
+      });
+
+      let response = await request(app).get(
+        `/entities/by-query?cursor=${Buffer.from(
+          JSON.stringify({ bad: 'cursor' }),
+          'utf8',
+        ).toString('base64')}`,
+      );
+      expect(response.status).toEqual(400);
+      expect(response.body.error.message).toMatch(/Malformed cursor/);
+
+      response = await request(app).get(`/entities/by-query?cursor=badcursor`);
+      expect(response.status).toEqual(400);
+      expect(response.body.error.message).toMatch(/Malformed cursor/);
+    });
+  });
+
   describe('GET /entities/by-uid/:uid', () => {
     it('can fetch entity by uid', async () => {
       const entity: Entity = {
@@ -560,6 +674,7 @@ describe('createRouter readonly enabled', () => {
       removeEntityByUid: jest.fn(),
       entityAncestry: jest.fn(),
       facets: jest.fn(),
+      queryEntities: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),
@@ -750,6 +865,7 @@ describe('NextRouter permissioning', () => {
       removeEntityByUid: jest.fn(),
       entityAncestry: jest.fn(),
       facets: jest.fn(),
+      queryEntities: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),
@@ -820,3 +936,12 @@ describe('NextRouter permissioning', () => {
     });
   });
 });
+
+function mockCursor(partialCursor?: Partial<Cursor>): Cursor {
+  return {
+    orderFields: [],
+    orderFieldValues: [],
+    isPrevious: false,
+    ...partialCursor,
+  };
+}

plugins/catalog-backend/src/service/createRouter.ts

@@ -39,12 +39,14 @@ import {
   parseEntityFilterParams,
   parseEntityPaginationParams,
   parseEntityTransformParams,
+  parseQueryEntitiesParams,
 } from './request';
 import { parseEntityFacetParams } from './request/parseEntityFacetParams';
 import { parseEntityOrderParams } from './request/parseEntityOrderParams';
 import { LocationService, RefreshOptions, RefreshService } from './types';
 import {
   disallowReadonlyMode,
+  encodeCursor,
   locationInput,
   validateRequestBody,
 } from './util';
@@ -130,6 +132,26 @@ export async function createRouter(
         // TODO(freben): encode the pageInfo in the response
         res.json(entities);
       })
+      .get('/entities/by-query', async (req, res) => {
+        const { items, pageInfo, totalItems } =
+          await entitiesCatalog.queryEntities({
+            ...parseQueryEntitiesParams(req.query),
+            authorizationToken: getBearerToken(req.header('authorization')),
+          });
+
+        res.json({
+          items,
+          totalItems,
+          pageInfo: {
+            ...(pageInfo.nextCursor && {
+              nextCursor: encodeCursor(pageInfo.nextCursor),
+            }),
+            ...(pageInfo.prevCursor && {
+              prevCursor: encodeCursor(pageInfo.prevCursor),
+            }),
+          },
+        });
+      })
       .get('/entities/by-uid/:uid', async (req, res) => {
         const { uid } = req.params;
         const { entities } = await entitiesCatalog.entities({

plugins/catalog-backend/src/service/request/index.ts

@@ -19,3 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter';
 export { parseEntityFilterParams } from './parseEntityFilterParams';
 export { parseEntityPaginationParams } from './parseEntityPaginationParams';
 export { parseEntityTransformParams } from './parseEntityTransformParams';
+export { parseQueryEntitiesParams } from './parseQueryEntitiesParams';

plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams';
+
+describe('parseEntityOrderFieldParams', () => {
+  it('supports no order fields', () => {
+    const result = parseEntityOrderFieldParams({});
+    expect(result).toEqual(undefined);
+  });
+
+  it('supports single orderField', () => {
+    const result = parseEntityOrderFieldParams({
+      orderField: ['metadata.name,desc'],
+    })!;
+    expect(result).toEqual([{ field: 'metadata.name', order: 'desc' }]);
+  });
+
+  it('supports single orderField without order', () => {
+    const result = parseEntityOrderFieldParams({
+      orderField: ['metadata.name'],
+    })!;
+    expect(result).toEqual([{ field: 'metadata.name' }]);
+  });
+
+  it('supports multiple order fields', () => {
+    const result = parseEntityOrderFieldParams({
+      orderField: ['metadata.name,desc', 'metadata.uid,asc'],
+    });
+
+    expect(result).toEqual([
+      { field: 'metadata.name', order: 'desc' },
+      { field: 'metadata.uid', order: 'asc' },
+    ]);
+  });
+
+  it('throws if orderField order is not valid', () => {
+    expect(() =>
+      parseEntityOrderFieldParams({
+        orderField: ['metadata.name,desc', 'metadata.uid,invalid'],
+      }),
+    ).toThrow(/Invalid order field order/);
+  });
+});

plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { InputError } from '@backstage/errors';
+import { EntityOrder } from '../../catalog/types';
+import { parseStringsParam } from './common';
+
+export function parseEntityOrderFieldParams(
+  params: Record<string, unknown>,
+): EntityOrder[] | undefined {
+  const orderFieldStrings = parseStringsParam(params.orderField, 'orderField');
+  if (!orderFieldStrings) {
+    return undefined;
+  }
+
+  return orderFieldStrings.map(orderFieldString => {
+    const [field, order] = orderFieldString.split(',');
+
+    if (order !== undefined && !isOrder(order)) {
+      throw new InputError('Invalid order field order, must be asc or desc');
+    }
+    return { field, order };
+  });
+}
+
+export function isOrder(order: string): order is 'asc' | 'desc' {
+  return ['asc', 'desc'].includes(order);
+}

plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts

@@ -0,0 +1,28 @@
+import { parseStringParam } from './common';
+
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function parseFullTextFilterFields(params: Record<string, unknown>) {
+  const fullTextFilterFields = parseStringParam(
+    params.fullTextFilterFields,
+    'fullTextFilterFields',
+  );
+  if (!fullTextFilterFields) {
+    return undefined;
+  }
+
+  return fullTextFilterFields.split(',');
+}

plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Cursor,
+  QueryEntitiesCursorRequest,
+  QueryEntitiesInitialRequest,
+} from '../../catalog/types';
+import { encodeCursor } from '../util';
+import { parseQueryEntitiesParams } from './parseQueryEntitiesParams';
+
+describe('parseQueryEntitiesParams', () => {
+  describe('initial request', () => {
+    it('should parse all the defined params', () => {
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        filter: ['a=1', 'b=2'],
+        orderField: ['metadata.name,desc'],
+        fullTextFilterTerm: 'query',
+        fullTextFilterFields: 'metadata.name,metadata.namespace',
+      };
+      const parsedObj = parseQueryEntitiesParams(
+        validRequest,
+      ) as QueryEntitiesInitialRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.orderFields).toEqual([
+        { field: 'metadata.name', order: 'desc' },
+      ]);
+      expect(parsedObj.filter).toBeDefined();
+      expect(parsedObj.fullTextFilter).toEqual({
+        term: 'query',
+        fields: ['metadata.name', 'metadata.namespace'],
+      });
+      expect(parsedObj).not.toHaveProperty('authorizationToken');
+      expect(parsedObj).not.toHaveProperty('cursor');
+    });
+    it('should ignore optional params', () => {
+      const parsedObj = parseQueryEntitiesParams(
+        {},
+      ) as QueryEntitiesInitialRequest;
+      expect(parsedObj.limit).toBeUndefined();
+      expect(parsedObj.fields).toBeUndefined();
+      expect(parsedObj.orderFields).toBeUndefined();
+      expect(parsedObj.filter).toBeUndefined();
+      expect(parsedObj.fullTextFilter).toEqual({ term: '', fields: undefined });
+      expect(parsedObj).not.toHaveProperty('authorizationToken');
+      expect(parsedObj).not.toHaveProperty('cursor');
+    });
+
+    it.each([
+      {
+        limit: 'asd',
+      },
+      { filter: 3 },
+      { orderField: ['metadata.uid,diagonal'] },
+      { fields: [4] },
+      { fullTextFilterTerm: [] },
+      { fullTextFilterFields: 3 },
+    ])('should throw if some parameter is not valid %p', params => {
+      expect(() => parseQueryEntitiesParams(params)).toThrow();
+    });
+  });
+
+  describe('cursor request', () => {
+    it('should parse all the defined params', () => {
+      const cursor: Cursor = {
+        totalItems: 100,
+        orderFields: [],
+        orderFieldValues: [],
+        isPrevious: false,
+      };
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        cursor: encodeCursor(cursor),
+      };
+      const parsedObj = parseQueryEntitiesParams(
+        validRequest,
+      ) as QueryEntitiesCursorRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.cursor).toEqual(cursor);
+    });
+
+    it('should ignore unknown params', () => {
+      const cursor: Cursor = {
+        totalItems: 100,
+        orderFields: [],
+        orderFieldValues: [],
+        isPrevious: false,
+      };
+      const validRequest = {
+        authorizationToken: 'to_not_be_returned',
+        fields: ['kind'],
+        limit: '3',
+        cursor: encodeCursor(cursor),
+        filter: ['a=1', 'b=2'],
+        orderField: 'orderField,desc',
+        query: 'query',
+      };
+      const parsedObj = parseQueryEntitiesParams(
+        validRequest,
+      ) as QueryEntitiesCursorRequest;
+      expect(parsedObj.limit).toBe(3);
+      expect(parsedObj.fields).toBeDefined();
+      expect(parsedObj.cursor).toEqual(cursor);
+      expect(parsedObj).not.toHaveProperty('filter');
+      expect(parsedObj).not.toHaveProperty('orderField');
+      expect(parsedObj).not.toHaveProperty('query');
+    });
+
+    it('should ignore optional params', () => {
+      const parsedObj = parseQueryEntitiesParams(
+        {},
+      ) as QueryEntitiesCursorRequest;
+      expect(parsedObj.limit).toBeUndefined();
+      expect(parsedObj.fields).toBeUndefined();
+    });
+
+    it.each([
+      {
+        limit: 'asd',
+      },
+      { cursor: [] },
+      { fields: [4] },
+    ])('should throw if some parameter is not valid %p', params => {
+      expect(() => parseQueryEntitiesParams(params)).toThrow();
+    });
+  });
+});

plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  QueryEntitiesCursorRequest,
+  QueryEntitiesInitialRequest,
+  QueryEntitiesRequest,
+} from '../../catalog/types';
+import { decodeCursor } from '../util';
+import { parseIntegerParam, parseStringParam } from './common';
+import { parseEntityFilterParams } from './parseEntityFilterParams';
+import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams';
+import { parseEntityTransformParams } from './parseEntityTransformParams';
+import { parseFullTextFilterFields } from './parseFullTextFilterFields';
+
+export function parseQueryEntitiesParams(
+  params: Record<string, unknown>,
+): Omit<QueryEntitiesRequest, 'authorizationToken'> {
+  const fields = parseEntityTransformParams(params);
+  const limit = parseIntegerParam(params.limit, 'limit');
+  const cursor = parseStringParam(params.cursor, 'cursor');
+  if (cursor) {
+    const decodedCursor = decodeCursor(cursor);
+    const response: Omit<QueryEntitiesCursorRequest, 'authorizationToken'> = {
+      cursor: decodedCursor,
+      fields,
+      limit,
+    };
+    return response;
+  }
+
+  const filter = parseEntityFilterParams(params);
+  const fullTextFilterTerm = parseStringParam(
+    params.fullTextFilterTerm,
+    'fullTextFilterTerm',
+  );
+  const fullTextFilterFields = parseFullTextFilterFields(params);
+
+  const orderFields = parseEntityOrderFieldParams(params);
+
+  const response: Omit<QueryEntitiesInitialRequest, 'authorizationToken'> = {
+    fields,
+    filter,
+    limit,
+    orderFields,
+    fullTextFilter: {
+      term: fullTextFilterTerm || '',
+      fields: fullTextFilterFields,
+    },
+  };
+
+  return response;
+}

plugins/catalog-backend/src/service/util.ts

@@ -18,6 +18,13 @@ import { InputError, NotAllowedError } from '@backstage/errors';
 import { Request } from 'express';
 import lodash from 'lodash';
 import { z } from 'zod';
+import {
+  Cursor,
+  EntityFilter,
+  QueryEntitiesCursorRequest,
+  QueryEntitiesInitialRequest,
+  QueryEntitiesRequest,
+} from '../catalog/types';
 
 export async function requireRequestBody(req: Request): Promise<unknown> {
   const contentType = req.header('content-type');
@@ -65,3 +72,63 @@ export function disallowReadonlyMode(readonly: boolean) {
     throw new NotAllowedError('This operation not allowed in readonly mode');
   }
 }
+
+export function isQueryEntitiesInitialRequest(
+  input: QueryEntitiesRequest | undefined,
+): input is QueryEntitiesInitialRequest {
+  if (!input) {
+    return false;
+  }
+  return !isQueryEntitiesCursorRequest(input);
+}
+
+export function isQueryEntitiesCursorRequest(
+  input: QueryEntitiesRequest | undefined,
+): input is QueryEntitiesCursorRequest {
+  if (!input) {
+    return false;
+  }
+  return !!(input as QueryEntitiesCursorRequest).cursor;
+}
+
+const entityFilterParser: z.ZodSchema<EntityFilter> = z.lazy(() =>
+  z
+    .object({
+      key: z.string(),
+      values: z.array(z.string()).optional(),
+    })
+    .or(z.object({ not: entityFilterParser }))
+    .or(z.object({ anyOf: z.array(entityFilterParser) }))
+    .or(z.object({ allOf: z.array(entityFilterParser) })),
+);
+
+export const cursorParser: z.ZodSchema<Cursor> = z.object({
+  orderFields: z.array(
+    z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }),
+  ),
+  orderFieldValues: z.array(z.string().or(z.null())),
+  filter: entityFilterParser.optional(),
+  isPrevious: z.boolean(),
+  query: z.string().optional(),
+  firstSortFieldValues: z.array(z.string().or(z.null())).optional(),
+  totalItems: z.number().optional(),
+});
+
+export function encodeCursor(cursor: Cursor) {
+  const json = JSON.stringify(cursor);
+  return Buffer.from(json, 'utf8').toString('base64');
+}
+
+export function decodeCursor(encodedCursor: string) {
+  try {
+    const data = Buffer.from(encodedCursor, 'base64').toString('utf8');
+    const result = cursorParser.safeParse(JSON.parse(data));
+
+    if (!result.success) {
+      throw new InputError(`Malformed cursor: ${result.error}`);
+    }
+    return result.data;
+  } catch (e) {
+    throw new InputError(`Malformed cursor: ${e}`);
+  }
+}