From ec3930d7555936914d487e90b43a71f96b673baf Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 7 Oct 2021 16:04:48 +0200 Subject: [PATCH 1/3] Catalog: Deprecate legacy catalog classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- plugins/catalog-backend/api-report.md | 10 +-- plugins/catalog-backend/src/catalog/index.ts | 5 -- plugins/catalog-backend/src/catalog/types.ts | 49 ++------------- .../catalog/DatabaseLocationsCatalog.test.ts | 2 +- .../catalog/DatabaseLocationsCatalog.ts | 5 +- .../src/legacy/catalog/index.ts | 7 +++ .../src/legacy/catalog/types.ts | 62 +++++++++++++++++++ .../ingestion/HigherOrderOperations.test.ts | 7 ++- .../legacy/ingestion/HigherOrderOperations.ts | 19 +++--- .../src/legacy/service/CatalogBuilder.ts | 6 +- .../src/legacy/service/router.test.ts | 6 +- .../src/legacy/service/router.ts | 5 +- .../src/next/NextCatalogBuilder.ts | 8 +-- 13 files changed, 109 insertions(+), 82 deletions(-) rename plugins/catalog-backend/src/{ => legacy}/catalog/DatabaseLocationsCatalog.test.ts (98%) rename plugins/catalog-backend/src/{ => legacy}/catalog/DatabaseLocationsCatalog.ts (94%) create mode 100644 plugins/catalog-backend/src/legacy/catalog/types.ts diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 68d50911d6..2aecebdda7 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -596,7 +596,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { // Warning: (ae-missing-release-tag) "DatabaseLocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export class DatabaseLocationsCatalog implements LocationsCatalog { constructor(database: Database); // (undocumented) @@ -1157,7 +1157,7 @@ export class LocationReaders implements LocationReader { // Warning: (ae-missing-release-tag) "LocationResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type LocationResponse = { data: Location_2; currentStatus: LocationUpdateStatus; @@ -1165,7 +1165,7 @@ export type LocationResponse = { // Warning: (ae-missing-release-tag) "LocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type LocationsCatalog = { addLocation(location: Location_2): Promise; removeLocation(id: string): Promise; @@ -1220,7 +1220,7 @@ export interface LocationStore { // Warning: (ae-missing-release-tag) "LocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type LocationUpdateLogEvent = { id: string; status: 'fail' | 'success'; @@ -1232,7 +1232,7 @@ export type LocationUpdateLogEvent = { // Warning: (ae-missing-release-tag) "LocationUpdateStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type LocationUpdateStatus = { timestamp: string | null; status: string | null; diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index f95d16a69f..7ac0150eef 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; export type { EntitiesCatalog, EntitiesRequest, @@ -22,9 +21,5 @@ export type { EntityAncestryResponse, EntityUpsertRequest, EntityUpsertResponse, - LocationResponse, - LocationsCatalog, - LocationUpdateLogEvent, - LocationUpdateStatus, PageInfo, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 217e077434..cd2590a26b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model'; +import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; import { EntityFilter, EntityPagination } from '../database/types'; // @@ -41,11 +41,13 @@ export type EntitiesResponse = { pageInfo: PageInfo; }; +/** @deprecated This was part of the legacy catalog engine */ export type EntityUpsertRequest = { entity: Entity; relations: EntityRelationSpec[]; }; +/** @deprecated This was part of the legacy catalog engine */ export type EntityUpsertResponse = { entityId: string; entity?: Entity; @@ -79,12 +81,14 @@ export type EntitiesCatalog = { /** * Writes a number of entities efficiently to storage. * + * @deprecated This method was part of the legacy catalog engine an will be removed. + * * @param requests - The entities and their relations * @param options.locationId - The location that they all belong to (default none) * @param options.dryRun - Whether to throw away the results (default false) * @param options.outputEntities - Whether to return the resulting entities (default false) */ - batchAddOrUpdateEntities( + batchAddOrUpdateEntities?( requests: EntityUpsertRequest[], options?: { locationId?: string; @@ -100,44 +104,3 @@ export type EntitiesCatalog = { */ entityAncestry(entityRef: string): Promise; }; - -// -// Locations -// - -export type LocationUpdateStatus = { - timestamp: string | null; - status: string | null; - message: string | null; -}; - -export type LocationUpdateLogEvent = { - id: string; - status: 'fail' | 'success'; - location_id: string; - entity_name: string; - created_at?: string; - message?: string; -}; - -export type LocationResponse = { - data: Location; - currentStatus: LocationUpdateStatus; -}; - -export type LocationsCatalog = { - addLocation(location: Location): Promise; - removeLocation(id: string): Promise; - locations(): Promise; - location(id: string): Promise; - locationHistory(id: string): Promise; - logUpdateSuccess( - locationId: string, - entityName?: string | string[], - ): Promise; - logUpdateFailure( - locationId: string, - error?: Error, - entityName?: string, - ): Promise; -}; diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts similarity index 98% rename from plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts rename to plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts index 8bd8e86b8e..24bcc79db1 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DatabaseManager } from '../database'; +import { DatabaseManager } from '../../database'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; const bootstrapLocation = { diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts similarity index 94% rename from plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts rename to plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts index e81c72a3da..4e3b6975a4 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts @@ -15,13 +15,14 @@ */ import { Location } from '@backstage/catalog-model'; -import type { Database } from '../database'; +import type { Database } from '../../database'; import { DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, -} from '../database/types'; +} from '../../database/types'; import { LocationResponse, LocationsCatalog } from './types'; +/** @deprecated This was part of the legacy catalog engine */ export class DatabaseLocationsCatalog implements LocationsCatalog { constructor(private readonly database: Database) {} diff --git a/plugins/catalog-backend/src/legacy/catalog/index.ts b/plugins/catalog-backend/src/legacy/catalog/index.ts index 237eb4c08e..96828d03d1 100644 --- a/plugins/catalog-backend/src/legacy/catalog/index.ts +++ b/plugins/catalog-backend/src/legacy/catalog/index.ts @@ -15,3 +15,10 @@ */ export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; +export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +export type { + LocationResponse, + LocationsCatalog, + LocationUpdateLogEvent, + LocationUpdateStatus, +} from './types'; diff --git a/plugins/catalog-backend/src/legacy/catalog/types.ts b/plugins/catalog-backend/src/legacy/catalog/types.ts new file mode 100644 index 0000000000..7ca1f3620a --- /dev/null +++ b/plugins/catalog-backend/src/legacy/catalog/types.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Location } from '@backstage/catalog-model'; + +// +// Locations +// + +/** @deprecated This was part of the legacy catalog engine */ +export type LocationUpdateStatus = { + timestamp: string | null; + status: string | null; + message: string | null; +}; + +/** @deprecated This was part of the legacy catalog engine */ +export type LocationUpdateLogEvent = { + id: string; + status: 'fail' | 'success'; + location_id: string; + entity_name: string; + created_at?: string; + message?: string; +}; + +/** @deprecated This was part of the legacy catalog engine */ +export type LocationResponse = { + data: Location; + currentStatus: LocationUpdateStatus; +}; + +/** @deprecated This was part of the legacy catalog engine */ +export type LocationsCatalog = { + addLocation(location: Location): Promise; + removeLocation(id: string): Promise; + locations(): Promise; + location(id: string): Promise; + locationHistory(id: string): Promise; + logUpdateSuccess( + locationId: string, + entityName?: string | string[], + ): Promise; + logUpdateFailure( + locationId: string, + error?: Error, + entityName?: string, + ): Promise; +}; diff --git a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts index 3f7fccf024..5508b4d8a2 100644 --- a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts @@ -16,14 +16,15 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; -import { EntitiesCatalog, LocationsCatalog } from '../../catalog'; -import { LocationUpdateStatus } from '../../catalog/types'; +import { EntitiesCatalog } from '../../catalog'; +import { LocationsCatalog } from '../catalog'; +import { LocationUpdateStatus } from '../catalog/types'; import { DatabaseLocationUpdateLogStatus } from '../../database/types'; import { HigherOrderOperations } from './HigherOrderOperations'; import { LocationReader } from './types'; describe('HigherOrderOperations', () => { - let entitiesCatalog: jest.Mocked; + let entitiesCatalog: jest.Mocked>; let locationsCatalog: jest.Mocked; let locationReader: jest.Mocked; let higherOrderOperation: HigherOrderOperations; diff --git a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts index 8d7ab6ff7c..9189b91fe8 100644 --- a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.ts @@ -21,7 +21,8 @@ import { } from '@backstage/catalog-model'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; -import { EntitiesCatalog, LocationsCatalog } from '../../catalog'; +import { EntitiesCatalog } from '../../catalog'; +import { LocationsCatalog } from '../catalog'; import { durationText } from '../../util'; import { AddLocationResult, @@ -95,14 +96,12 @@ export class HigherOrderOperations implements HigherOrderOperation { return { location, entities: [] }; } - const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities( - readerOutput.entities, - { - locationId: dryRun ? undefined : location.id, - dryRun, - outputEntities: true, - }, - ); + const writtenEntities = await this.entitiesCatalog + .batchAddOrUpdateEntities!(readerOutput.entities, { + locationId: dryRun ? undefined : location.id, + dryRun, + outputEntities: true, + }); const entities = writtenEntities.map(e => e.entity!); @@ -186,7 +185,7 @@ export class HigherOrderOperations implements HigherOrderOperation { startTimestamp = process.hrtime(); try { - await this.entitiesCatalog.batchAddOrUpdateEntities( + await this.entitiesCatalog.batchAddOrUpdateEntities!( readerOutput.entities, { locationId: location.id }, ); diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index c5b3281f9a..dee096ce7d 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -26,12 +26,12 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrations } from '@backstage/integration'; import lodash from 'lodash'; +import { EntitiesCatalog } from '../../catalog'; import { + DatabaseEntitiesCatalog, DatabaseLocationsCatalog, - EntitiesCatalog, LocationsCatalog, -} from '../../catalog'; -import { DatabaseEntitiesCatalog } from '../catalog'; +} from '../catalog'; import { DatabaseManager } from '../../database'; import { AnnotateLocationEntityProcessor, diff --git a/plugins/catalog-backend/src/legacy/service/router.test.ts b/plugins/catalog-backend/src/legacy/service/router.test.ts index 3fc655d761..d15d3799b5 100644 --- a/plugins/catalog-backend/src/legacy/service/router.test.ts +++ b/plugins/catalog-backend/src/legacy/service/router.test.ts @@ -20,15 +20,15 @@ import { NotFoundError } from '@backstage/errors'; import type { Entity, LocationSpec } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; -import { EntitiesCatalog, LocationsCatalog } from '../../catalog'; -import { LocationResponse } from '../../catalog/types'; +import { EntitiesCatalog } from '../../catalog'; +import { LocationResponse, LocationsCatalog } from '../catalog/types'; import { HigherOrderOperation } from '../ingestion/types'; import { createRouter } from './router'; import { basicEntityFilter } from '../../service/request'; import { RefreshService } from '../../next'; describe('createRouter readonly disabled', () => { - let entitiesCatalog: jest.Mocked; + let entitiesCatalog: jest.Mocked>; let locationsCatalog: jest.Mocked; let higherOrderOperation: jest.Mocked; let app: express.Express; diff --git a/plugins/catalog-backend/src/legacy/service/router.ts b/plugins/catalog-backend/src/legacy/service/router.ts index 651880720e..4dfa9e49f9 100644 --- a/plugins/catalog-backend/src/legacy/service/router.ts +++ b/plugins/catalog-backend/src/legacy/service/router.ts @@ -26,7 +26,8 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import yn from 'yn'; -import { EntitiesCatalog, LocationsCatalog } from '../../catalog'; +import { EntitiesCatalog } from '../../catalog'; +import { LocationsCatalog } from '../catalog'; import { LocationAnalyzer } from '../../ingestion/types'; import { HigherOrderOperation } from '../ingestion/types'; import { @@ -124,7 +125,7 @@ export async function createRouter( disallowReadonlyMode(readonlyEnabled); const body = await requireRequestBody(req); - const [result] = await entitiesCatalog.batchAddOrUpdateEntities([ + const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([ { entity: body as Entity, relations: [] }, ]); const response = await entitiesCatalog.entities({ diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 615f3c55e1..77c87f38ff 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -33,11 +33,8 @@ import { ScmIntegrations } from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; import lodash from 'lodash'; -import { - DatabaseLocationsCatalog, - EntitiesCatalog, - LocationsCatalog, -} from '../catalog'; +import { EntitiesCatalog } from '../catalog'; +import { DatabaseLocationsCatalog, LocationsCatalog } from '../legacy/catalog'; import { CommonDatabase } from '../database/CommonDatabase'; import { AnnotateLocationEntityProcessor, @@ -289,6 +286,7 @@ export class NextCatalogBuilder { */ async build(): Promise<{ entitiesCatalog: EntitiesCatalog; + /** @deprecated This will be removed */ locationsCatalog: LocationsCatalog; locationAnalyzer: LocationAnalyzer; processingEngine: CatalogProcessingEngine; From f5df64c82295782c8fb148dc6f3600e91536355c Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 7 Oct 2021 17:59:00 +0200 Subject: [PATCH 2/3] feat: moved the deprecated database implemmentation to legacy catalog Signed-off-by: blam Co-authored-by: Johan Haals Co-authored-by: Patrik Oldsberg --- plugins/catalog-backend/api-report.md | 58 +++++++++--------- plugins/catalog-backend/src/catalog/index.ts | 3 + plugins/catalog-backend/src/catalog/types.ts | 48 +++++++++++++-- plugins/catalog-backend/src/index.ts | 1 - .../catalog/DatabaseEntitiesCatalog.test.ts | 2 +- .../legacy/catalog/DatabaseEntitiesCatalog.ts | 4 +- .../catalog/DatabaseLocationsCatalog.test.ts | 2 +- .../catalog/DatabaseLocationsCatalog.ts | 4 +- .../database/CommonDatabase.test.ts | 2 +- .../{ => legacy}/database/CommonDatabase.ts | 5 +- .../{ => legacy}/database/DatabaseManager.ts | 2 + .../src/{ => legacy}/database/index.ts | 3 - .../src/{ => legacy}/database/search.test.ts | 0 .../src/{ => legacy}/database/search.ts | 0 .../src/{ => legacy}/database/types.ts | 60 +++++-------------- plugins/catalog-backend/src/legacy/index.ts | 1 + .../ingestion/HigherOrderOperations.test.ts | 2 +- .../src/legacy/service/CatalogBuilder.test.ts | 2 +- .../src/legacy/service/CatalogBuilder.ts | 2 +- .../src/next/NextCatalogBuilder.ts | 8 ++- .../src/next/NextEntitiesCatalog.ts | 3 +- .../src/next/database/DatabaseManager.ts | 3 +- .../database/DefaultProcessingDatabase.ts | 21 +++---- .../src/next/database/metrics.ts | 3 +- .../src/next/database/tables.ts | 9 +++ .../src/next/database/types.ts | 8 ++- .../src/service/request/basicEntityFilter.ts | 2 +- .../request/parseEntityFilterParams.ts | 2 +- .../request/parseEntityPaginationParams.ts | 2 +- .../src/service/standaloneServer.ts | 2 +- 30 files changed, 148 insertions(+), 116 deletions(-) rename plugins/catalog-backend/src/{ => legacy}/database/CommonDatabase.test.ts (99%) rename plugins/catalog-backend/src/{ => legacy}/database/CommonDatabase.ts (99%) rename plugins/catalog-backend/src/{ => legacy}/database/DatabaseManager.ts (96%) rename plugins/catalog-backend/src/{ => legacy}/database/index.ts (94%) rename plugins/catalog-backend/src/{ => legacy}/database/search.test.ts (100%) rename plugins/catalog-backend/src/{ => legacy}/database/search.ts (100%) rename plugins/catalog-backend/src/{ => legacy}/database/types.ts (85%) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 2aecebdda7..6b7d966b28 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -435,7 +435,7 @@ export class CodeOwnersProcessor implements CatalogProcessor { // Warning: (ae-missing-release-tag) "CommonDatabase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public +// @public @deprecated export class CommonDatabase implements Database { constructor(database: Knex, logger: Logger_2); // (undocumented) @@ -499,7 +499,7 @@ export class CommonDatabase implements Database { // Warning: (ae-missing-release-tag) "CreateDatabaseOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type CreateDatabaseOptions = { logger: Logger_2; }; @@ -528,7 +528,7 @@ export function createRouter(options: RouterOptions): Promise; // Warning: (ae-missing-release-tag) "Database" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public +// @public @deprecated export type Database = { transaction(fn: (tx: Transaction) => Promise): Promise; addEntities( @@ -624,7 +624,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { // Warning: (ae-missing-release-tag) "DatabaseLocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DatabaseLocationUpdateLogEvent = { id: string; status: DatabaseLocationUpdateLogStatus; @@ -646,7 +646,7 @@ export enum DatabaseLocationUpdateLogStatus { // Warning: (ae-missing-release-tag) "DatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export class DatabaseManager { // (undocumented) static createDatabase( @@ -665,7 +665,7 @@ export class DatabaseManager { // Warning: (ae-missing-release-tag) "DbEntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbEntitiesRequest = { filter?: EntityFilter; pagination?: EntityPagination; @@ -673,7 +673,7 @@ export type DbEntitiesRequest = { // Warning: (ae-missing-release-tag) "DbEntitiesResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbEntitiesResponse = { entities: DbEntityResponse[]; pageInfo: DbPageInfo; @@ -681,7 +681,7 @@ export type DbEntitiesResponse = { // Warning: (ae-missing-release-tag) "DbEntityRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbEntityRequest = { locationId?: string; entity: Entity; @@ -690,7 +690,7 @@ export type DbEntityRequest = { // Warning: (ae-missing-release-tag) "DbEntityResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbEntityResponse = { locationId?: string; entity: Entity; @@ -698,7 +698,7 @@ export type DbEntityResponse = { // Warning: (ae-missing-release-tag) "DbLocationsRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbLocationsRow = { id: string; type: string; @@ -707,7 +707,7 @@ export type DbLocationsRow = { // Warning: (ae-missing-release-tag) "DbLocationsRowWithStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbLocationsRowWithStatus = DbLocationsRow & { status: string | null; timestamp: string | null; @@ -716,7 +716,7 @@ export type DbLocationsRowWithStatus = DbLocationsRow & { // Warning: (ae-missing-release-tag) "DbPageInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type DbPageInfo = | { hasNextPage: false; @@ -812,7 +812,7 @@ export function durationText(startTimestamp: [number, number]): string; export type EntitiesCatalog = { entities(request?: EntitiesRequest): Promise; removeEntityByUid(uid: string): Promise; - batchAddOrUpdateEntities( + batchAddOrUpdateEntities?( requests: EntityUpsertRequest[], options?: { locationId?: string; @@ -943,7 +943,7 @@ export type EntityProviderMutation = // Warning: (ae-missing-release-tag) "EntityUpsertRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type EntityUpsertRequest = { entity: Entity; relations: EntityRelationSpec[]; @@ -951,7 +951,7 @@ export type EntityUpsertRequest = { // Warning: (ae-missing-release-tag) "EntityUpsertResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public @deprecated (undocumented) export type EntityUpsertResponse = { entityId: string; entity?: Entity; @@ -1487,7 +1487,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor { // Warning: (ae-missing-release-tag) "Transaction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public +// @public @deprecated export type Transaction = { rollback(): Promise; }; @@ -1512,20 +1512,20 @@ export class UrlReaderProcessor implements CatalogProcessor { // Warnings were encountered during analysis: // -// src/catalog/types.d.ts:52:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/catalog/types.d.ts:53:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/catalog/types.d.ts:54:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:146:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:147:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:148:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:150:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:163:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:164:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/database/types.d.ts:165:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/catalog/types.d.ts:97:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// src/catalog/types.d.ts:98:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// src/catalog/types.d.ts:99:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts // src/ingestion/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:98:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:104:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:105:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:119:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:120:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:121:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:123:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:136:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:137:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// src/legacy/database/types.d.ts:138:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/legacy/ingestion/types.d.ts:19:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen ``` diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index 7ac0150eef..885a2f5389 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -22,4 +22,7 @@ export type { EntityUpsertRequest, EntityUpsertResponse, PageInfo, + EntitiesSearchFilter, + EntityFilter, + EntityPagination, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index cd2590a26b..0bc0ec4311 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,11 +15,51 @@ */ import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; -import { EntityFilter, EntityPagination } from '../database/types'; -// -// Entities -// +/** + * A filter expression for entities. + * + * Any (at least one) of the outer sets must match, within which all of the + * individual filters must match. + */ +export type EntityFilter = { + anyOf: { allOf: EntitiesSearchFilter[] }[]; +}; + +/** + * A pagination rule for entities. + */ +export type EntityPagination = { + limit?: number; + offset?: number; + after?: string; +}; + +/** + * Matches rows in the entities_search table. + */ +export type EntitiesSearchFilter = { + /** + * The key to match on. + * + * Matches are always case insensitive. + */ + key: string; + + /** + * Match on plain equality of values. + * + * If undefined, this factor is not taken into account. Otherwise, match on + * values that are equal to any of the given array items. Matches are always + * case insensitive. + */ + matchValueIn?: string[]; + + /** + * Match on existence of key. + */ + matchValueExists?: boolean; +}; export type PageInfo = | { diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index af0c32f73b..4d84330f27 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -21,7 +21,6 @@ */ export * from './catalog'; -export * from './database'; export * from './ingestion'; export * from './legacy'; export * from './search'; diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts index 953e08d07a..e2db8a0fc0 100644 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.test.ts @@ -16,7 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model'; -import { Database, DatabaseManager, Transaction } from '../../database'; +import { Database, DatabaseManager, Transaction } from '../database'; import { basicEntityFilter } from '../../service/request'; import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; import { EntityUpsertRequest } from '../../catalog/types'; diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts index f4f8fd320e..8776a5a4dc 100644 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseEntitiesCatalog.ts @@ -26,8 +26,8 @@ import { ConflictError } from '@backstage/errors'; import { chunk, groupBy } from 'lodash'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; -import type { Database, DbEntityResponse, Transaction } from '../../database'; -import { DbEntitiesRequest } from '../../database/types'; +import type { Database, DbEntityResponse, Transaction } from '../database'; +import { DbEntitiesRequest } from '../database/types'; import { basicEntityFilter } from '../../service/request'; import { durationText } from '../../util/timing'; import type { diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts index 24bcc79db1..8bd8e86b8e 100644 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DatabaseManager } from '../../database'; +import { DatabaseManager } from '../database'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; const bootstrapLocation = { diff --git a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts index 4e3b6975a4..1273e193b2 100644 --- a/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/legacy/catalog/DatabaseLocationsCatalog.ts @@ -15,11 +15,11 @@ */ import { Location } from '@backstage/catalog-model'; -import type { Database } from '../../database'; +import type { Database } from '../database'; import { DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, -} from '../../database/types'; +} from '../database/types'; import { LocationResponse, LocationsCatalog } from './types'; /** @deprecated This was part of the legacy catalog engine */ diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts similarity index 99% rename from plugins/catalog-backend/src/database/CommonDatabase.test.ts rename to plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts index f459cbc446..ca2e6c2934 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/legacy/database/CommonDatabase.test.ts @@ -16,7 +16,7 @@ import { Entity, Location, parseEntityRef } from '@backstage/catalog-model'; import { ConflictError } from '@backstage/errors'; -import { basicEntityFilter } from '../service/request'; +import { basicEntityFilter } from '../../service/request'; import { DatabaseManager } from './DatabaseManager'; import type { DbEntityRequest, diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/legacy/database/CommonDatabase.ts similarity index 99% rename from plugins/catalog-backend/src/database/CommonDatabase.ts rename to plugins/catalog-backend/src/legacy/database/CommonDatabase.ts index 79cad3bd23..5cc31b5be5 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/legacy/database/CommonDatabase.ts @@ -44,9 +44,9 @@ import { DbLocationsRow, DbLocationsRowWithStatus, DbPageInfo, - EntityPagination, Transaction, } from './types'; +import { EntityPagination } from '../../catalog/types'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -55,7 +55,8 @@ import { const BATCH_SIZE = 50; /** - * The core database implementation. + * The core database implementation.. + * @deprecated This was part of the legacy catalog engin */ export class CommonDatabase implements Database { constructor( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/legacy/database/DatabaseManager.ts similarity index 96% rename from plugins/catalog-backend/src/database/DatabaseManager.ts rename to plugins/catalog-backend/src/legacy/database/DatabaseManager.ts index 2e43290fb5..9bb75a1043 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/legacy/database/DatabaseManager.ts @@ -26,6 +26,7 @@ const migrationsDir = resolvePackagePath( 'migrations', ); +/** @deprecated This was part of the legacy catalog engine */ export type CreateDatabaseOptions = { logger: Logger; }; @@ -34,6 +35,7 @@ const defaultOptions: CreateDatabaseOptions = { logger: getVoidLogger(), }; +/** @deprecated This was part of the legacy catalog engine */ export class DatabaseManager { public static async createDatabase( knex: Knex, diff --git a/plugins/catalog-backend/src/database/index.ts b/plugins/catalog-backend/src/legacy/database/index.ts similarity index 94% rename from plugins/catalog-backend/src/database/index.ts rename to plugins/catalog-backend/src/legacy/database/index.ts index 028b0caa4b..909c1de26c 100644 --- a/plugins/catalog-backend/src/database/index.ts +++ b/plugins/catalog-backend/src/legacy/database/index.ts @@ -21,9 +21,6 @@ export type { Database, DbEntityRequest, DbEntityResponse, - EntitiesSearchFilter, - EntityFilter, - EntityPagination, Transaction, DbEntitiesRequest, DbEntitiesResponse, diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/legacy/database/search.test.ts similarity index 100% rename from plugins/catalog-backend/src/database/search.test.ts rename to plugins/catalog-backend/src/legacy/database/search.test.ts diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/legacy/database/search.ts similarity index 100% rename from plugins/catalog-backend/src/database/search.ts rename to plugins/catalog-backend/src/legacy/database/search.ts diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/legacy/database/types.ts similarity index 85% rename from plugins/catalog-backend/src/database/types.ts rename to plugins/catalog-backend/src/legacy/database/types.ts index e1a253decc..b68681b4bd 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/legacy/database/types.ts @@ -20,7 +20,9 @@ import type { EntityRelationSpec, Location, } from '@backstage/catalog-model'; +import { EntityFilter, EntityPagination } from '../../catalog/types'; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntitiesRow = { id: string; location_id: string | null; @@ -30,22 +32,26 @@ export type DbEntitiesRow = { data: string; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntityRequest = { locationId?: string; entity: Entity; relations: EntityRelationSpec[]; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntitiesRequest = { filter?: EntityFilter; pagination?: EntityPagination; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntitiesResponse = { entities: DbEntityResponse[]; pageInfo: DbPageInfo; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbPageInfo = | { hasNextPage: false; @@ -55,11 +61,13 @@ export type DbPageInfo = endCursor: string; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntityResponse = { locationId?: string; entity: Entity; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntitiesRelationsRow = { originating_entity_id: string; source_full_name: string; @@ -67,18 +75,21 @@ export type DbEntitiesRelationsRow = { target_full_name: string; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbEntitiesSearchRow = { entity_id: string; key: string; value: string | null; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbLocationsRow = { id: string; type: string; target: string; }; +/** @deprecated This was part of the legacy catalog engine */ export type DbLocationsRowWithStatus = DbLocationsRow & { status: string | null; timestamp: string | null; @@ -90,6 +101,7 @@ export enum DatabaseLocationUpdateLogStatus { SUCCESS = 'success', } +/** @deprecated This was part of the legacy catalog engine */ export type DatabaseLocationUpdateLogEvent = { id: string; status: DatabaseLocationUpdateLogStatus; @@ -99,53 +111,10 @@ export type DatabaseLocationUpdateLogEvent = { message?: string; }; -/** - * Matches rows in the entities_search table. - */ -export type EntitiesSearchFilter = { - /** - * The key to match on. - * - * Matches are always case insensitive. - */ - key: string; - - /** - * Match on plain equality of values. - * - * If undefined, this factor is not taken into account. Otherwise, match on - * values that are equal to any of the given array items. Matches are always - * case insensitive. - */ - matchValueIn?: string[]; - - /** - * Match on existence of key. - */ - matchValueExists?: boolean; -}; - -/** - * A filter expression for entities. - * - * Any (at least one) of the outer sets must match, within which all of the - * individual filters must match. - */ -export type EntityFilter = { - anyOf: { allOf: EntitiesSearchFilter[] }[]; -}; - -/** - * A pagination rule for entities. - */ -export type EntityPagination = { - limit?: number; - offset?: number; - after?: string; -}; - /** * An abstraction for transactions of the underlying database technology. + * + * @deprecated This was part of the legacy catalog engine */ export type Transaction = { rollback(): Promise; @@ -154,6 +123,7 @@ export type Transaction = { /** * An abstraction on top of the underlying database, wrapping the basic CRUD * needs. + * @deprecated This was part of the legacy catalog engine */ export type Database = { /** diff --git a/plugins/catalog-backend/src/legacy/index.ts b/plugins/catalog-backend/src/legacy/index.ts index 74074aa319..be8c7b8d63 100644 --- a/plugins/catalog-backend/src/legacy/index.ts +++ b/plugins/catalog-backend/src/legacy/index.ts @@ -17,3 +17,4 @@ export * from './catalog'; export * from './ingestion'; export * from './service'; +export * from './database'; diff --git a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts index 5508b4d8a2..e6c042421b 100644 --- a/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/legacy/ingestion/HigherOrderOperations.test.ts @@ -19,7 +19,7 @@ import { Entity, Location, LocationSpec } from '@backstage/catalog-model'; import { EntitiesCatalog } from '../../catalog'; import { LocationsCatalog } from '../catalog'; import { LocationUpdateStatus } from '../catalog/types'; -import { DatabaseLocationUpdateLogStatus } from '../../database/types'; +import { DatabaseLocationUpdateLogStatus } from '../database/types'; import { HigherOrderOperations } from './HigherOrderOperations'; import { LocationReader } from './types'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts index 5d3a4dd326..1b366130ed 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.test.ts @@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { Knex } from 'knex'; import yaml from 'yaml'; -import { DatabaseManager } from '../../database'; +import { DatabaseManager } from '../database'; import { CatalogProcessorParser } from '../../ingestion'; import * as result from '../../ingestion/processors/results'; import { CatalogBuilder } from './CatalogBuilder'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index dee096ce7d..1a151b77d6 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -32,7 +32,7 @@ import { DatabaseLocationsCatalog, LocationsCatalog, } from '../catalog'; -import { DatabaseManager } from '../../database'; +import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 77c87f38ff..700c5a5163 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -34,8 +34,12 @@ import { createHash } from 'crypto'; import { Router } from 'express'; import lodash from 'lodash'; import { EntitiesCatalog } from '../catalog'; -import { DatabaseLocationsCatalog, LocationsCatalog } from '../legacy/catalog'; -import { CommonDatabase } from '../database/CommonDatabase'; +import { + DatabaseLocationsCatalog, + LocationsCatalog, + CommonDatabase, +} from '../legacy'; + import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index 14fda4ea24..4fde078a8b 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -22,13 +22,14 @@ import { EntitiesRequest, EntitiesResponse, EntityAncestryResponse, + EntityPagination, } from '../catalog/types'; -import { DbPageInfo, EntityPagination } from '../database/types'; import { DbFinalEntitiesRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbSearchRow, + DbPageInfo, } from './database/tables'; function parsePagination(input?: EntityPagination): { diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts index 124e8227b7..3b31542025 100644 --- a/plugins/catalog-backend/src/next/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/next/database/DatabaseManager.ts @@ -18,8 +18,7 @@ import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; import knexFactory, { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; -import { CommonDatabase } from '../../database/CommonDatabase'; -import { Database } from '../../database/types'; +import { CommonDatabase, Database } from '../../legacy'; export type CreateDatabaseOptions = { logger: Logger; diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 7b32bec5e9..b47154a46f 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -20,17 +20,8 @@ import { Knex } from 'knex'; import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; -import { Transaction } from '../../database'; -import { DeferredEntity } from '../processing/types'; -import { RefreshIntervalFunction } from '../refresh'; -import { rethrowError, timestampToDateTime } from './conversion'; -import { initDatabaseMetrics } from './metrics'; -import { - DbRefreshStateReferencesRow, - DbRefreshStateRow, - DbRelationsRow, -} from './tables'; import { + Transaction, GetProcessableEntitiesResult, ProcessingDatabase, RefreshStateItem, @@ -41,6 +32,16 @@ import { ListAncestorsResult, UpdateEntityCacheOptions, } from './types'; +import { DeferredEntity } from '../processing/types'; +import { RefreshIntervalFunction } from '../refresh'; +import { rethrowError, timestampToDateTime } from './conversion'; +import { initDatabaseMetrics } from './metrics'; +import { + DbRefreshStateReferencesRow, + DbRefreshStateRow, + DbRelationsRow, +} from './tables'; + import { generateStableHash } from './util'; // The number of items that are sent per batch to the database layer, when diff --git a/plugins/catalog-backend/src/next/database/metrics.ts b/plugins/catalog-backend/src/next/database/metrics.ts index a41ad512f0..4ad319ed10 100644 --- a/plugins/catalog-backend/src/next/database/metrics.ts +++ b/plugins/catalog-backend/src/next/database/metrics.ts @@ -15,9 +15,8 @@ */ import { Knex } from 'knex'; -import { DbLocationsRow } from '../../database/types'; import { createGaugeMetric } from '../metrics'; -import { DbRefreshStateRow, DbRelationsRow } from './tables'; +import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; export function initDatabaseMetrics(knex: Knex) { const seen = new Set(); diff --git a/plugins/catalog-backend/src/next/database/tables.ts b/plugins/catalog-backend/src/next/database/tables.ts index e064b17d7b..40b4cb9a12 100644 --- a/plugins/catalog-backend/src/next/database/tables.ts +++ b/plugins/catalog-backend/src/next/database/tables.ts @@ -14,6 +14,15 @@ * limitations under the License. */ +export type DbPageInfo = + | { + hasNextPage: false; + } + | { + hasNextPage: true; + endCursor: string; + }; + export type DbLocationsRow = { id: string; type: string; diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 3f1a4784fb..041a3fdc5b 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -17,9 +17,15 @@ import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { DateTime } from 'luxon'; -import { Transaction } from '../../database/types'; import { DeferredEntity } from '../processing/types'; +/** + * An abstraction for transactions of the underlying database technology. + */ +export type Transaction = { + rollback(): Promise; +}; + export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { diff --git a/plugins/catalog-backend/src/service/request/basicEntityFilter.ts b/plugins/catalog-backend/src/service/request/basicEntityFilter.ts index a3ad1c2f73..1cf9ca95f0 100644 --- a/plugins/catalog-backend/src/service/request/basicEntityFilter.ts +++ b/plugins/catalog-backend/src/service/request/basicEntityFilter.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntitiesSearchFilter, EntityFilter } from '../../database'; +import { EntitiesSearchFilter, EntityFilter } from '../../catalog'; /** * Forms a full EntityFilter based on a single key-value(s) object. diff --git a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts index c63b366bc7..452958b7ae 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts @@ -15,7 +15,7 @@ */ import { InputError } from '@backstage/errors'; -import { EntitiesSearchFilter, EntityFilter } from '../../database'; +import { EntitiesSearchFilter, EntityFilter } from '../../catalog'; import { parseStringsParam } from './common'; /** diff --git a/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts index c22dcd5935..c91c0b0214 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts @@ -15,7 +15,7 @@ */ import { InputError } from '@backstage/errors'; -import { EntityPagination } from '../../database'; +import { EntityPagination } from '../../catalog'; import { parseIntegerParam, parseStringParam } from './common'; /** diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 4a3409802f..7aae3cd47c 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -22,7 +22,7 @@ import { } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; -import { DatabaseManager } from '../database'; +import { DatabaseManager } from '../legacy/database'; import { CatalogBuilder } from '../legacy/service/CatalogBuilder'; import { createRouter } from '../legacy/service'; From 590605bf3d2b8042ec6839d93942ce58756bc0c1 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 7 Oct 2021 18:20:55 +0200 Subject: [PATCH 3/3] chore: removed old DatabaseManager from Catalog Next Signed-off-by: blam Co-authored-by: Johan Haals Co-authored-by: Patrik Oldsberg --- .../src/next/DefaultLocationStore.test.ts | 4 +- .../src/next/DefaultRefreshService.test.ts | 4 +- .../src/next/NextCatalogBuilder.ts | 14 +-- .../src/next/NextEntitiesCatalog.test.ts | 4 +- .../src/next/database/DatabaseManager.ts | 98 ------------------- .../DefaultProcessingDatabase.test.ts | 4 +- .../src/next/database/migrations.ts | 29 ++++++ .../src/next/stitching/Stitcher.test.ts | 4 +- 8 files changed, 42 insertions(+), 119 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/database/DatabaseManager.ts create mode 100644 plugins/catalog-backend/src/next/database/migrations.ts diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts index 82b08f2e70..f3cefeb14f 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; -import { DatabaseManager } from './database/DatabaseManager'; +import { applyDatabaseMigrations } from './database/migrations'; import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { @@ -25,7 +25,7 @@ describe('DefaultLocationStore', () => { async function createLocationStore(databaseId: TestDatabaseId) { const knex = await databases.init(databaseId); - await DatabaseManager.createDatabase(knex); + await applyDatabaseMigrations(knex); const connection = { applyMutation: jest.fn() }; const store = new DefaultLocationStore(knex); await store.connect(connection); diff --git a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts index 79ced3ea43..7b6a181375 100644 --- a/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/next/DefaultRefreshService.test.ts @@ -19,7 +19,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { createHash } from 'crypto'; import { Knex } from 'knex'; import { Logger } from 'winston'; -import { DatabaseManager } from './database/DatabaseManager'; +import { applyDatabaseMigrations } from './database/migrations'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { DbRefreshStateReferencesRow, @@ -44,7 +44,7 @@ describe('Refresh integration', () => { logger: Logger = defaultLogger, ) { const knex = await databases.init(databaseId); - await DatabaseManager.createDatabase(knex); + await applyDatabaseMigrations(knex); return { knex, db: new DefaultProcessingDatabase({ diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 700c5a5163..8ca93d4bd7 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - PluginDatabaseManager, - resolvePackagePath, - UrlReader, -} from '@backstage/backend-common'; +import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; import { DefaultNamespaceEntityPolicy, EntityPolicies, @@ -70,6 +66,7 @@ import { } from '../next/types'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; +import { applyDatabaseMigrations } from './database/migrations'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; import { DefaultLocationStore } from './DefaultLocationStore'; @@ -304,12 +301,7 @@ export class NextCatalogBuilder { const parser = this.parser || defaultEntityDataParser; const dbClient = await database.getClient(); - await dbClient.migrate.latest({ - directory: resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrations', - ), - }); + await applyDatabaseMigrations(dbClient); const db = new CommonDatabase(dbClient, logger); diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts index c173c926e8..47e92445e9 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts @@ -18,7 +18,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DatabaseManager } from './database/DatabaseManager'; +import { applyDatabaseMigrations } from './database/migrations'; import { DbFinalEntitiesRow, DbRefreshStateReferencesRow, @@ -33,7 +33,7 @@ describe('NextEntitiesCatalog', () => { async function createDatabase(databaseId: TestDatabaseId) { const knex = await databases.init(databaseId); - await DatabaseManager.createDatabase(knex); + await applyDatabaseMigrations(knex); return { knex }; } diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts deleted file mode 100644 index 3b31542025..0000000000 --- a/plugins/catalog-backend/src/next/database/DatabaseManager.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; -import knexFactory, { Knex } from 'knex'; -import { v4 as uuid } from 'uuid'; -import { Logger } from 'winston'; -import { CommonDatabase, Database } from '../../legacy'; - -export type CreateDatabaseOptions = { - logger: Logger; -}; - -const defaultOptions: CreateDatabaseOptions = { - logger: getVoidLogger(), -}; - -export class DatabaseManager { - public static async createDatabase( - knex: Knex, - options: Partial = {}, - ): Promise { - const migrationsDir = resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrations', - ); - - await knex.migrate.latest({ - directory: migrationsDir, - }); - const { logger } = { ...defaultOptions, ...options }; - return new CommonDatabase(knex, logger); - } - - public static async createInMemoryDatabase(): Promise { - const knex = await this.createInMemoryDatabaseConnection(); - return await this.createDatabase(knex); - } - - public static async createInMemoryDatabaseConnection(): Promise { - const knex = knexFactory({ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }); - - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - - return knex; - } - - public static async createTestDatabase(): Promise { - const knex = await this.createTestDatabaseConnection(); - return await this.createDatabase(knex); - } - - public static async createTestDatabaseConnection(): Promise { - const config: Knex.Config = { - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }; - - let knex = knexFactory(config); - if (typeof config.connection !== 'string') { - const tempDbName = `d${uuid().replace(/-/g, '')}`; - await knex.raw(`CREATE DATABASE ${tempDbName};`); - knex = knexFactory({ - ...config, - connection: { - ...config.connection, - database: tempDbName, - }, - }); - } - - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - - return knex; - } -} diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 6148325581..5f509747f3 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -21,7 +21,7 @@ import { Knex } from 'knex'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { DateTime } from 'luxon'; -import { DatabaseManager } from './DatabaseManager'; +import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { DbRefreshStateReferencesRow, @@ -43,7 +43,7 @@ describe('Default Processing Database', () => { logger: Logger = defaultLogger, ) { const knex = await databases.init(databaseId); - await DatabaseManager.createDatabase(knex); + await applyDatabaseMigrations(knex); return { knex, db: new DefaultProcessingDatabase({ diff --git a/plugins/catalog-backend/src/next/database/migrations.ts b/plugins/catalog-backend/src/next/database/migrations.ts new file mode 100644 index 0000000000..ed3a7751b5 --- /dev/null +++ b/plugins/catalog-backend/src/next/database/migrations.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; + +export async function applyDatabaseMigrations(knex: Knex): Promise { + const migrationsDir = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrations', + ); + + await knex.migrate.latest({ + directory: migrationsDir, + }); +} diff --git a/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts index 96af65fffb..be57ff7083 100644 --- a/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; -import { DatabaseManager } from '../database/DatabaseManager'; +import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, DbRefreshStateReferencesRow, @@ -37,7 +37,7 @@ describe('Stitcher', () => { 'runs the happy path for %p', async databaseId => { const db = await databases.init(databaseId); - await DatabaseManager.createDatabase(db); + await applyDatabaseMigrations(db); const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[];