Merge pull request #7500 from backstage/mob/legacy-catalog

catalog: Continued deprecation of legacy catalog internals
This commit is contained in:
Johan Haals
2021-10-08 13:23:47 +02:00
committed by GitHub
41 changed files with 288 additions and 306 deletions
+34 -34
View File
@@ -435,7 +435,7 @@ export class CodeOwnersProcessor implements CatalogProcessor {
// Warning: (ae-missing-release-tag) "CommonDatabase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
// @public @deprecated
export class CommonDatabase implements Database {
constructor(database: Knex, logger: Logger_2);
// (undocumented)
@@ -499,7 +499,7 @@ export class CommonDatabase implements Database {
// Warning: (ae-missing-release-tag) "CreateDatabaseOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type CreateDatabaseOptions = {
logger: Logger_2;
};
@@ -528,7 +528,7 @@ export function createRouter(options: RouterOptions): Promise<express.Router>;
// Warning: (ae-missing-release-tag) "Database" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
// @public @deprecated
export type Database = {
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
addEntities(
@@ -596,7 +596,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
// Warning: (ae-missing-release-tag) "DatabaseLocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(database: Database);
// (undocumented)
@@ -624,7 +624,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog {
// Warning: (ae-missing-release-tag) "DatabaseLocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DatabaseLocationUpdateLogEvent = {
id: string;
status: DatabaseLocationUpdateLogStatus;
@@ -646,7 +646,7 @@ export enum DatabaseLocationUpdateLogStatus {
// Warning: (ae-missing-release-tag) "DatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export class DatabaseManager {
// (undocumented)
static createDatabase(
@@ -665,7 +665,7 @@ export class DatabaseManager {
// Warning: (ae-missing-release-tag) "DbEntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
@@ -673,7 +673,7 @@ export type DbEntitiesRequest = {
// Warning: (ae-missing-release-tag) "DbEntitiesResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
@@ -681,7 +681,7 @@ export type DbEntitiesResponse = {
// Warning: (ae-missing-release-tag) "DbEntityRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbEntityRequest = {
locationId?: string;
entity: Entity;
@@ -690,7 +690,7 @@ export type DbEntityRequest = {
// Warning: (ae-missing-release-tag) "DbEntityResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
@@ -698,7 +698,7 @@ export type DbEntityResponse = {
// Warning: (ae-missing-release-tag) "DbLocationsRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbLocationsRow = {
id: string;
type: string;
@@ -707,7 +707,7 @@ export type DbLocationsRow = {
// Warning: (ae-missing-release-tag) "DbLocationsRowWithStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbLocationsRowWithStatus = DbLocationsRow & {
status: string | null;
timestamp: string | null;
@@ -716,7 +716,7 @@ export type DbLocationsRowWithStatus = DbLocationsRow & {
// Warning: (ae-missing-release-tag) "DbPageInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type DbPageInfo =
| {
hasNextPage: false;
@@ -812,7 +812,7 @@ export function durationText(startTimestamp: [number, number]): string;
export type EntitiesCatalog = {
entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
removeEntityByUid(uid: string): Promise<void>;
batchAddOrUpdateEntities(
batchAddOrUpdateEntities?(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
@@ -943,7 +943,7 @@ export type EntityProviderMutation =
// Warning: (ae-missing-release-tag) "EntityUpsertRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
@@ -951,7 +951,7 @@ export type EntityUpsertRequest = {
// Warning: (ae-missing-release-tag) "EntityUpsertResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type EntityUpsertResponse = {
entityId: string;
entity?: Entity;
@@ -1157,7 +1157,7 @@ export class LocationReaders implements LocationReader {
// Warning: (ae-missing-release-tag) "LocationResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type LocationResponse = {
data: Location_2;
currentStatus: LocationUpdateStatus;
@@ -1165,7 +1165,7 @@ export type LocationResponse = {
// Warning: (ae-missing-release-tag) "LocationsCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type LocationsCatalog = {
addLocation(location: Location_2): Promise<Location_2>;
removeLocation(id: string): Promise<void>;
@@ -1220,7 +1220,7 @@ export interface LocationStore {
// Warning: (ae-missing-release-tag) "LocationUpdateLogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type LocationUpdateLogEvent = {
id: string;
status: 'fail' | 'success';
@@ -1232,7 +1232,7 @@ export type LocationUpdateLogEvent = {
// Warning: (ae-missing-release-tag) "LocationUpdateStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
// @public @deprecated (undocumented)
export type LocationUpdateStatus = {
timestamp: string | null;
status: string | null;
@@ -1487,7 +1487,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor {
// Warning: (ae-missing-release-tag) "Transaction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
// @public @deprecated
export type Transaction = {
rollback(): Promise<unknown>;
};
@@ -1512,20 +1512,20 @@ export class UrlReaderProcessor implements CatalogProcessor {
// Warnings were encountered during analysis:
//
// src/catalog/types.d.ts:52:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/catalog/types.d.ts:53:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/catalog/types.d.ts:54:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:146:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:147:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:148:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:150:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:163:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:164:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/database/types.d.ts:165:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/catalog/types.d.ts:97:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/catalog/types.d.ts:98:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/catalog/types.d.ts:99:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
// src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts
// src/ingestion/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:98:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:104:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:105:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:119:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:120:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:121:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:123:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:136:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:137:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/database/types.d.ts:138:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// src/legacy/ingestion/types.d.ts:19:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
```
+3 -5
View File
@@ -14,7 +14,6 @@
* limitations under the License.
*/
export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
export type {
EntitiesCatalog,
EntitiesRequest,
@@ -22,9 +21,8 @@ export type {
EntityAncestryResponse,
EntityUpsertRequest,
EntityUpsertResponse,
LocationResponse,
LocationsCatalog,
LocationUpdateLogEvent,
LocationUpdateStatus,
PageInfo,
EntitiesSearchFilter,
EntityFilter,
EntityPagination,
} from './types';
+50 -47
View File
@@ -14,12 +14,52 @@
* limitations under the License.
*/
import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
import { EntityFilter, EntityPagination } from '../database/types';
import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
//
// Entities
//
/**
* A filter expression for entities.
*
* Any (at least one) of the outer sets must match, within which all of the
* individual filters must match.
*/
export type EntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
/**
* A pagination rule for entities.
*/
export type EntityPagination = {
limit?: number;
offset?: number;
after?: string;
};
/**
* Matches rows in the entities_search table.
*/
export type EntitiesSearchFilter = {
/**
* The key to match on.
*
* Matches are always case insensitive.
*/
key: string;
/**
* Match on plain equality of values.
*
* If undefined, this factor is not taken into account. Otherwise, match on
* values that are equal to any of the given array items. Matches are always
* case insensitive.
*/
matchValueIn?: string[];
/**
* Match on existence of key.
*/
matchValueExists?: boolean;
};
export type PageInfo =
| {
@@ -41,11 +81,13 @@ export type EntitiesResponse = {
pageInfo: PageInfo;
};
/** @deprecated This was part of the legacy catalog engine */
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
};
/** @deprecated This was part of the legacy catalog engine */
export type EntityUpsertResponse = {
entityId: string;
entity?: Entity;
@@ -79,12 +121,14 @@ export type EntitiesCatalog = {
/**
* Writes a number of entities efficiently to storage.
*
* @deprecated This method was part of the legacy catalog engine an will be removed.
*
* @param requests - The entities and their relations
* @param options.locationId - The location that they all belong to (default none)
* @param options.dryRun - Whether to throw away the results (default false)
* @param options.outputEntities - Whether to return the resulting entities (default false)
*/
batchAddOrUpdateEntities(
batchAddOrUpdateEntities?(
requests: EntityUpsertRequest[],
options?: {
locationId?: string;
@@ -100,44 +144,3 @@ export type EntitiesCatalog = {
*/
entityAncestry(entityRef: string): Promise<EntityAncestryResponse>;
};
//
// Locations
//
export type LocationUpdateStatus = {
timestamp: string | null;
status: string | null;
message: string | null;
};
export type LocationUpdateLogEvent = {
id: string;
status: 'fail' | 'success';
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
export type LocationResponse = {
data: Location;
currentStatus: LocationUpdateStatus;
};
export type LocationsCatalog = {
addLocation(location: Location): Promise<Location>;
removeLocation(id: string): Promise<void>;
locations(): Promise<LocationResponse[]>;
location(id: string): Promise<LocationResponse>;
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void>;
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
};
-1
View File
@@ -21,7 +21,6 @@
*/
export * from './catalog';
export * from './database';
export * from './ingestion';
export * from './legacy';
export * from './search';
@@ -16,7 +16,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
import { Database, DatabaseManager, Transaction } from '../../database';
import { Database, DatabaseManager, Transaction } from '../database';
import { basicEntityFilter } from '../../service/request';
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
import { EntityUpsertRequest } from '../../catalog/types';
@@ -26,8 +26,8 @@ import { ConflictError } from '@backstage/errors';
import { chunk, groupBy } from 'lodash';
import limiterFactory from 'p-limit';
import { Logger } from 'winston';
import type { Database, DbEntityResponse, Transaction } from '../../database';
import { DbEntitiesRequest } from '../../database/types';
import type { Database, DbEntityResponse, Transaction } from '../database';
import { DbEntitiesRequest } from '../database/types';
import { basicEntityFilter } from '../../service/request';
import { durationText } from '../../util/timing';
import type {
@@ -22,6 +22,7 @@ import {
} from '../database/types';
import { LocationResponse, LocationsCatalog } from './types';
/** @deprecated This was part of the legacy catalog engine */
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(private readonly database: Database) {}
@@ -15,3 +15,10 @@
*/
export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
export type {
LocationResponse,
LocationsCatalog,
LocationUpdateLogEvent,
LocationUpdateStatus,
} from './types';
@@ -0,0 +1,62 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Location } from '@backstage/catalog-model';
//
// Locations
//
/** @deprecated This was part of the legacy catalog engine */
export type LocationUpdateStatus = {
timestamp: string | null;
status: string | null;
message: string | null;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationUpdateLogEvent = {
id: string;
status: 'fail' | 'success';
location_id: string;
entity_name: string;
created_at?: string;
message?: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationResponse = {
data: Location;
currentStatus: LocationUpdateStatus;
};
/** @deprecated This was part of the legacy catalog engine */
export type LocationsCatalog = {
addLocation(location: Location): Promise<Location>;
removeLocation(id: string): Promise<void>;
locations(): Promise<LocationResponse[]>;
location(id: string): Promise<LocationResponse>;
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
logUpdateSuccess(
locationId: string,
entityName?: string | string[],
): Promise<void>;
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
};
@@ -16,7 +16,7 @@
import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
import { ConflictError } from '@backstage/errors';
import { basicEntityFilter } from '../service/request';
import { basicEntityFilter } from '../../service/request';
import { DatabaseManager } from './DatabaseManager';
import type {
DbEntityRequest,
@@ -44,9 +44,9 @@ import {
DbLocationsRow,
DbLocationsRowWithStatus,
DbPageInfo,
EntityPagination,
Transaction,
} from './types';
import { EntityPagination } from '../../catalog/types';
// The number of items that are sent per batch to the database layer, when
// doing .batchInsert calls to knex. This needs to be low enough to not cause
@@ -55,7 +55,8 @@ import {
const BATCH_SIZE = 50;
/**
* The core database implementation.
* The core database implementation..
* @deprecated This was part of the legacy catalog engin
*/
export class CommonDatabase implements Database {
constructor(
@@ -26,6 +26,7 @@ const migrationsDir = resolvePackagePath(
'migrations',
);
/** @deprecated This was part of the legacy catalog engine */
export type CreateDatabaseOptions = {
logger: Logger;
};
@@ -34,6 +35,7 @@ const defaultOptions: CreateDatabaseOptions = {
logger: getVoidLogger(),
};
/** @deprecated This was part of the legacy catalog engine */
export class DatabaseManager {
public static async createDatabase(
knex: Knex,
@@ -21,9 +21,6 @@ export type {
Database,
DbEntityRequest,
DbEntityResponse,
EntitiesSearchFilter,
EntityFilter,
EntityPagination,
Transaction,
DbEntitiesRequest,
DbEntitiesResponse,
@@ -20,7 +20,9 @@ import type {
EntityRelationSpec,
Location,
} from '@backstage/catalog-model';
import { EntityFilter, EntityPagination } from '../../catalog/types';
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRow = {
id: string;
location_id: string | null;
@@ -30,22 +32,26 @@ export type DbEntitiesRow = {
data: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntityRequest = {
locationId?: string;
entity: Entity;
relations: EntityRelationSpec[];
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbPageInfo =
| {
hasNextPage: false;
@@ -55,11 +61,13 @@ export type DbPageInfo =
endCursor: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesRelationsRow = {
originating_entity_id: string;
source_full_name: string;
@@ -67,18 +75,21 @@ export type DbEntitiesRelationsRow = {
target_full_name: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbEntitiesSearchRow = {
entity_id: string;
key: string;
value: string | null;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbLocationsRow = {
id: string;
type: string;
target: string;
};
/** @deprecated This was part of the legacy catalog engine */
export type DbLocationsRowWithStatus = DbLocationsRow & {
status: string | null;
timestamp: string | null;
@@ -90,6 +101,7 @@ export enum DatabaseLocationUpdateLogStatus {
SUCCESS = 'success',
}
/** @deprecated This was part of the legacy catalog engine */
export type DatabaseLocationUpdateLogEvent = {
id: string;
status: DatabaseLocationUpdateLogStatus;
@@ -99,53 +111,10 @@ export type DatabaseLocationUpdateLogEvent = {
message?: string;
};
/**
* Matches rows in the entities_search table.
*/
export type EntitiesSearchFilter = {
/**
* The key to match on.
*
* Matches are always case insensitive.
*/
key: string;
/**
* Match on plain equality of values.
*
* If undefined, this factor is not taken into account. Otherwise, match on
* values that are equal to any of the given array items. Matches are always
* case insensitive.
*/
matchValueIn?: string[];
/**
* Match on existence of key.
*/
matchValueExists?: boolean;
};
/**
* A filter expression for entities.
*
* Any (at least one) of the outer sets must match, within which all of the
* individual filters must match.
*/
export type EntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
/**
* A pagination rule for entities.
*/
export type EntityPagination = {
limit?: number;
offset?: number;
after?: string;
};
/**
* An abstraction for transactions of the underlying database technology.
*
* @deprecated This was part of the legacy catalog engine
*/
export type Transaction = {
rollback(): Promise<unknown>;
@@ -154,6 +123,7 @@ export type Transaction = {
/**
* An abstraction on top of the underlying database, wrapping the basic CRUD
* needs.
* @deprecated This was part of the legacy catalog engine
*/
export type Database = {
/**
@@ -17,3 +17,4 @@
export * from './catalog';
export * from './ingestion';
export * from './service';
export * from './database';
@@ -16,14 +16,15 @@
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
import { LocationUpdateStatus } from '../../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../../database/types';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { LocationUpdateStatus } from '../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../database/types';
import { HigherOrderOperations } from './HigherOrderOperations';
import { LocationReader } from './types';
describe('HigherOrderOperations', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let locationReader: jest.Mocked<LocationReader>;
let higherOrderOperation: HigherOrderOperations;
@@ -21,7 +21,8 @@ import {
} from '@backstage/catalog-model';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { durationText } from '../../util';
import {
AddLocationResult,
@@ -95,14 +96,12 @@ export class HigherOrderOperations implements HigherOrderOperation {
return { location, entities: [] };
}
const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities(
readerOutput.entities,
{
locationId: dryRun ? undefined : location.id,
dryRun,
outputEntities: true,
},
);
const writtenEntities = await this.entitiesCatalog
.batchAddOrUpdateEntities!(readerOutput.entities, {
locationId: dryRun ? undefined : location.id,
dryRun,
outputEntities: true,
});
const entities = writtenEntities.map(e => e.entity!);
@@ -186,7 +185,7 @@ export class HigherOrderOperations implements HigherOrderOperation {
startTimestamp = process.hrtime();
try {
await this.entitiesCatalog.batchAddOrUpdateEntities(
await this.entitiesCatalog.batchAddOrUpdateEntities!(
readerOutput.entities,
{ locationId: location.id },
);
@@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { Knex } from 'knex';
import yaml from 'yaml';
import { DatabaseManager } from '../../database';
import { DatabaseManager } from '../database';
import { CatalogProcessorParser } from '../../ingestion';
import * as result from '../../ingestion/processors/results';
import { CatalogBuilder } from './CatalogBuilder';
@@ -26,13 +26,13 @@ import {
} from '@backstage/catalog-model';
import { ScmIntegrations } from '@backstage/integration';
import lodash from 'lodash';
import { EntitiesCatalog } from '../../catalog';
import {
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
EntitiesCatalog,
LocationsCatalog,
} from '../../catalog';
import { DatabaseEntitiesCatalog } from '../catalog';
import { DatabaseManager } from '../../database';
} from '../catalog';
import { DatabaseManager } from '../database';
import {
AnnotateLocationEntityProcessor,
BitbucketDiscoveryProcessor,
@@ -20,15 +20,15 @@ import { NotFoundError } from '@backstage/errors';
import type { Entity, LocationSpec } from '@backstage/catalog-model';
import express from 'express';
import request from 'supertest';
import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
import { LocationResponse } from '../../catalog/types';
import { EntitiesCatalog } from '../../catalog';
import { LocationResponse, LocationsCatalog } from '../catalog/types';
import { HigherOrderOperation } from '../ingestion/types';
import { createRouter } from './router';
import { basicEntityFilter } from '../../service/request';
import { RefreshService } from '../../next';
describe('createRouter readonly disabled', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let entitiesCatalog: jest.Mocked<Required<EntitiesCatalog>>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let higherOrderOperation: jest.Mocked<HigherOrderOperation>;
let app: express.Express;
@@ -26,7 +26,8 @@ import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog, LocationsCatalog } from '../../catalog';
import { EntitiesCatalog } from '../../catalog';
import { LocationsCatalog } from '../catalog';
import { LocationAnalyzer } from '../../ingestion/types';
import { HigherOrderOperation } from '../ingestion/types';
import {
@@ -124,7 +125,7 @@ export async function createRouter(
disallowReadonlyMode(readonlyEnabled);
const body = await requireRequestBody(req);
const [result] = await entitiesCatalog.batchAddOrUpdateEntities([
const [result] = await entitiesCatalog.batchAddOrUpdateEntities!([
{ entity: body as Entity, relations: [] },
]);
const response = await entitiesCatalog.entities({
@@ -15,7 +15,7 @@
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { v4 as uuid } from 'uuid';
import { DatabaseManager } from './database/DatabaseManager';
import { applyDatabaseMigrations } from './database/migrations';
import { DefaultLocationStore } from './DefaultLocationStore';
describe('DefaultLocationStore', () => {
@@ -25,7 +25,7 @@ describe('DefaultLocationStore', () => {
async function createLocationStore(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await DatabaseManager.createDatabase(knex);
await applyDatabaseMigrations(knex);
const connection = { applyMutation: jest.fn() };
const store = new DefaultLocationStore(knex);
await store.connect(connection);
@@ -19,7 +19,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { createHash } from 'crypto';
import { Knex } from 'knex';
import { Logger } from 'winston';
import { DatabaseManager } from './database/DatabaseManager';
import { applyDatabaseMigrations } from './database/migrations';
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import {
DbRefreshStateReferencesRow,
@@ -44,7 +44,7 @@ describe('Refresh integration', () => {
logger: Logger = defaultLogger,
) {
const knex = await databases.init(databaseId);
await DatabaseManager.createDatabase(knex);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
@@ -14,11 +14,7 @@
* limitations under the License.
*/
import {
PluginDatabaseManager,
resolvePackagePath,
UrlReader,
} from '@backstage/backend-common';
import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common';
import {
DefaultNamespaceEntityPolicy,
EntityPolicies,
@@ -33,12 +29,13 @@ import { ScmIntegrations } from '@backstage/integration';
import { createHash } from 'crypto';
import { Router } from 'express';
import lodash from 'lodash';
import { EntitiesCatalog } from '../catalog';
import {
DatabaseLocationsCatalog,
EntitiesCatalog,
LocationsCatalog,
} from '../catalog';
import { CommonDatabase } from '../database/CommonDatabase';
CommonDatabase,
} from '../legacy';
import {
AnnotateLocationEntityProcessor,
BitbucketDiscoveryProcessor,
@@ -69,6 +66,7 @@ import {
} from '../next/types';
import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider';
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
import { applyDatabaseMigrations } from './database/migrations';
import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine';
import { DefaultLocationService } from './DefaultLocationService';
import { DefaultLocationStore } from './DefaultLocationStore';
@@ -289,6 +287,7 @@ export class NextCatalogBuilder {
*/
async build(): Promise<{
entitiesCatalog: EntitiesCatalog;
/** @deprecated This will be removed */
locationsCatalog: LocationsCatalog;
locationAnalyzer: LocationAnalyzer;
processingEngine: CatalogProcessingEngine;
@@ -302,12 +301,7 @@ export class NextCatalogBuilder {
const parser = this.parser || defaultEntityDataParser;
const dbClient = await database.getClient();
await dbClient.migrate.latest({
directory: resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrations',
),
});
await applyDatabaseMigrations(dbClient);
const db = new CommonDatabase(dbClient, logger);
@@ -18,7 +18,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
import { DatabaseManager } from './database/DatabaseManager';
import { applyDatabaseMigrations } from './database/migrations';
import {
DbFinalEntitiesRow,
DbRefreshStateReferencesRow,
@@ -33,7 +33,7 @@ describe('NextEntitiesCatalog', () => {
async function createDatabase(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await DatabaseManager.createDatabase(knex);
await applyDatabaseMigrations(knex);
return { knex };
}
@@ -22,13 +22,14 @@ import {
EntitiesRequest,
EntitiesResponse,
EntityAncestryResponse,
EntityPagination,
} from '../catalog/types';
import { DbPageInfo, EntityPagination } from '../database/types';
import {
DbFinalEntitiesRow,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbSearchRow,
DbPageInfo,
} from './database/tables';
function parsePagination(input?: EntityPagination): {
@@ -1,99 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
import knexFactory, { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
import { Logger } from 'winston';
import { CommonDatabase } from '../../database/CommonDatabase';
import { Database } from '../../database/types';
export type CreateDatabaseOptions = {
logger: Logger;
};
const defaultOptions: CreateDatabaseOptions = {
logger: getVoidLogger(),
};
export class DatabaseManager {
public static async createDatabase(
knex: Knex,
options: Partial<CreateDatabaseOptions> = {},
): Promise<Database> {
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrations',
);
await knex.migrate.latest({
directory: migrationsDir,
});
const { logger } = { ...defaultOptions, ...options };
return new CommonDatabase(knex, logger);
}
public static async createInMemoryDatabase(): Promise<Database> {
const knex = await this.createInMemoryDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createInMemoryDatabaseConnection(): Promise<Knex> {
const knex = knexFactory({
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
});
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return knex;
}
public static async createTestDatabase(): Promise<Database> {
const knex = await this.createTestDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createTestDatabaseConnection(): Promise<Knex> {
const config: Knex.Config<any> = {
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
};
let knex = knexFactory(config);
if (typeof config.connection !== 'string') {
const tempDbName = `d${uuid().replace(/-/g, '')}`;
await knex.raw(`CREATE DATABASE ${tempDbName};`);
knex = knexFactory({
...config,
connection: {
...config.connection,
database: tempDbName,
},
});
}
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return knex;
}
}
@@ -21,7 +21,7 @@ import { Knex } from 'knex';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import { DateTime } from 'luxon';
import { DatabaseManager } from './DatabaseManager';
import { applyDatabaseMigrations } from './migrations';
import { DefaultProcessingDatabase } from './DefaultProcessingDatabase';
import {
DbRefreshStateReferencesRow,
@@ -43,7 +43,7 @@ describe('Default Processing Database', () => {
logger: Logger = defaultLogger,
) {
const knex = await databases.init(databaseId);
await DatabaseManager.createDatabase(knex);
await applyDatabaseMigrations(knex);
return {
knex,
db: new DefaultProcessingDatabase({
@@ -20,17 +20,8 @@ import { Knex } from 'knex';
import lodash from 'lodash';
import { v4 as uuid } from 'uuid';
import type { Logger } from 'winston';
import { Transaction } from '../../database';
import { DeferredEntity } from '../processing/types';
import { RefreshIntervalFunction } from '../refresh';
import { rethrowError, timestampToDateTime } from './conversion';
import { initDatabaseMetrics } from './metrics';
import {
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbRelationsRow,
} from './tables';
import {
Transaction,
GetProcessableEntitiesResult,
ProcessingDatabase,
RefreshStateItem,
@@ -41,6 +32,16 @@ import {
ListAncestorsResult,
UpdateEntityCacheOptions,
} from './types';
import { DeferredEntity } from '../processing/types';
import { RefreshIntervalFunction } from '../refresh';
import { rethrowError, timestampToDateTime } from './conversion';
import { initDatabaseMetrics } from './metrics';
import {
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbRelationsRow,
} from './tables';
import { generateStableHash } from './util';
// The number of items that are sent per batch to the database layer, when
@@ -15,9 +15,8 @@
*/
import { Knex } from 'knex';
import { DbLocationsRow } from '../../database/types';
import { createGaugeMetric } from '../metrics';
import { DbRefreshStateRow, DbRelationsRow } from './tables';
import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';
export function initDatabaseMetrics(knex: Knex) {
const seen = new Set<string>();
@@ -0,0 +1,29 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import { Knex } from 'knex';
export async function applyDatabaseMigrations(knex: Knex): Promise<void> {
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrations',
);
await knex.migrate.latest({
directory: migrationsDir,
});
}
@@ -14,6 +14,15 @@
* limitations under the License.
*/
export type DbPageInfo =
| {
hasNextPage: false;
}
| {
hasNextPage: true;
endCursor: string;
};
export type DbLocationsRow = {
id: string;
type: string;
@@ -17,9 +17,15 @@
import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
import { JsonObject } from '@backstage/config';
import { DateTime } from 'luxon';
import { Transaction } from '../../database/types';
import { DeferredEntity } from '../processing/types';
/**
* An abstraction for transactions of the underlying database technology.
*/
export type Transaction = {
rollback(): Promise<unknown>;
};
export type AddUnprocessedEntitiesResult = {};
export type UpdateProcessedEntityOptions = {
@@ -17,7 +17,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Entity } from '@backstage/catalog-model';
import { DatabaseManager } from '../database/DatabaseManager';
import { applyDatabaseMigrations } from '../database/migrations';
import {
DbFinalEntitiesRow,
DbRefreshStateReferencesRow,
@@ -37,7 +37,7 @@ describe('Stitcher', () => {
'runs the happy path for %p',
async databaseId => {
const db = await databases.init(databaseId);
await DatabaseManager.createDatabase(db);
await applyDatabaseMigrations(db);
const stitcher = new Stitcher(db, logger);
let entities: DbFinalEntitiesRow[];
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { EntitiesSearchFilter, EntityFilter } from '../../database';
import { EntitiesSearchFilter, EntityFilter } from '../../catalog';
/**
* Forms a full EntityFilter based on a single key-value(s) object.
@@ -15,7 +15,7 @@
*/
import { InputError } from '@backstage/errors';
import { EntitiesSearchFilter, EntityFilter } from '../../database';
import { EntitiesSearchFilter, EntityFilter } from '../../catalog';
import { parseStringsParam } from './common';
/**
@@ -15,7 +15,7 @@
*/
import { InputError } from '@backstage/errors';
import { EntityPagination } from '../../database';
import { EntityPagination } from '../../catalog';
import { parseIntegerParam, parseStringParam } from './common';
/**
@@ -22,7 +22,7 @@ import {
} from '@backstage/backend-common';
import { Server } from 'http';
import { Logger } from 'winston';
import { DatabaseManager } from '../database';
import { DatabaseManager } from '../legacy/database';
import { CatalogBuilder } from '../legacy/service/CatalogBuilder';
import { createRouter } from '../legacy/service';