paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

Merge pull request #15142 from IlyaSavich/catalog-collator-refactoring

Browse Source 
Refactor catalog collator
This commit is contained in:
Fredrik Adelöw
2023-01-25 10:47:58 +01:00
committed by GitHub
parent 3cb84125f0 860b1c0902
commit a0871a3c0a
10 changed files with 343 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/api-report.md
+10 -1
View File
@@ -167,6 +167,11 @@ export class CatalogBuilder {
useLegacySingleProcessorValidation(): this;
}
// @public (undocumented)
export type CatalogCollatorEntityTransformer = (
entity: Entity,
) => Omit<CatalogEntityDocument, 'location' | 'authorization'>;
// @alpha
export const catalogConditions: Conditions<{
hasAnnotation: PermissionRule<
@@ -350,6 +355,9 @@ export class DefaultCatalogCollator {
readonly visibilityPermission: Permission;
}
// @public (undocumented)
export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer;
// @public (undocumented)
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
@@ -360,7 +368,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
getCollator(): Promise<Readable>;
// (undocumented)
readonly type: string;
readonly type = 'software-catalog';
// (undocumented)
readonly visibilityPermission: Permission;
}
@@ -373,6 +381,7 @@ export type DefaultCatalogCollatorFactoryOptions = {
filter?: GetEntitiesRequest['filter'];
batchSize?: number;
catalogClient?: CatalogApi;
entityTransformer?: CatalogCollatorEntityTransformer;
};
export { DeferredEntity };
plugins/catalog-backend/src/search/util.ts → plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts
+6 -12
View File
@@ -14,16 +14,10 @@
* limitations under the License.
*/
import { Entity, isUserEntity, isGroupEntity } from '@backstage/catalog-model';
import { Entity } from '@backstage/catalog-model';
import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
export function getDocumentText(entity: Entity): string {
const documentTexts: string[] = [];
documentTexts.push(entity.metadata.description || '');
if (isUserEntity(entity) || isGroupEntity(entity)) {
if (entity.spec?.profile?.displayName) {
documentTexts.push(entity.spec.profile.displayName);
}
}
return documentTexts.join(' : ');
}
/** @public */
export type CatalogCollatorEntityTransformer = (
entity: Entity,
) => Omit<CatalogEntityDocument, 'location' | 'authorization'>;
plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts
+61 -7
View File
@@ -95,11 +95,6 @@ describe('DefaultCatalogCollatorFactory', () => {
);
});
it('has expected type', () => {
const factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
expect(factory.type).toBe('software-catalog');
});
describe('getCollator', () => {
let factory: DefaultCatalogCollatorFactory;
let collator: Readable;
@@ -124,24 +119,28 @@ describe('DefaultCatalogCollatorFactory', () => {
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents[0]).toMatchObject({
expect(documents[0]).toEqual({
title: expectedEntities[0].metadata.name,
location: '/catalog/default/component/test-entity',
text: expectedEntities[0].metadata.description,
namespace: 'default',
componentType: expectedEntities[0]!.spec!.type,
kind: expectedEntities[0]!.kind,
type: expectedEntities[0]!.spec!.type,
lifecycle: expectedEntities[0]!.spec!.lifecycle,
owner: expectedEntities[0]!.spec!.owner,
authorization: {
resourceRef: 'component:default/test-entity',
},
});
expect(documents[1]).toMatchObject({
expect(documents[1]).toEqual({
title: expectedEntities[1].metadata.title,
location: '/catalog/default/component/test-entity-2',
text: expectedEntities[1].metadata.description,
namespace: 'default',
componentType: expectedEntities[1]!.spec!.type,
kind: expectedEntities[1]!.kind,
type: expectedEntities[1]!.spec!.type,
lifecycle: expectedEntities[1]!.spec!.lifecycle,
owner: expectedEntities[1]!.spec!.owner,
authorization: {
@@ -150,6 +149,61 @@ describe('DefaultCatalogCollatorFactory', () => {
});
});
it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => {
const customFactory = DefaultCatalogCollatorFactory.fromConfig(config, {
...options,
entityTransformer: entity => ({
title: `custom-title-${
entity.metadata.title ?? entity.metadata.name
}`,
namespace: 'custom/namespace',
text: 'custom-text',
type: 'custom-type',
componentType: 'custom-component-type',
kind: 'custom-kind',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'custom:resource/ref',
},
location: '/custom/location',
}),
});
const customCollator = await customFactory.getCollator();
const pipeline = TestPipeline.fromCollator(customCollator);
const { documents } = await pipeline.execute();
expect(documents[0]).toEqual({
title: 'custom-title-test-entity',
location: '/catalog/default/component/test-entity',
text: 'custom-text',
namespace: 'custom/namespace',
componentType: 'custom-component-type',
kind: 'custom-kind',
type: 'custom-type',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'component:default/test-entity',
},
});
expect(documents[1]).toEqual({
title: 'custom-title-Test Entity',
location: '/catalog/default/component/test-entity-2',
text: 'custom-text',
namespace: 'custom/namespace',
componentType: 'custom-component-type',
kind: 'custom-kind',
type: 'custom-type',
lifecycle: 'custom-lifecycle',
owner: 'custom-owner',
authorization: {
resourceRef: 'component:default/test-entity-2',
},
});
});
it('maps a returned entity with a custom locationTemplate', async () => {
// Provide an alternate location template.
factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), {
plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts
+27 -26
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import { Config } from '@backstage/config';
import {
PluginEndpointDiscovery,
TokenManager,
@@ -23,8 +24,6 @@ import {
CatalogClient,
GetEntitiesRequest,
} from '@backstage/catalog-client';
import { stringifyEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import {
catalogEntityReadPermission,
@@ -32,7 +31,9 @@ import {
} from '@backstage/plugin-catalog-common';
import { Permission } from '@backstage/plugin-permission-common';
import { Readable } from 'stream';
import { getDocumentText } from './util';
import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
import { stringifyEntityRef } from '@backstage/catalog-model';
/** @public */
export type DefaultCatalogCollatorFactoryOptions = {
@@ -42,11 +43,12 @@ export type DefaultCatalogCollatorFactoryOptions = {
filter?: GetEntitiesRequest['filter'];
batchSize?: number;
catalogClient?: CatalogApi;
entityTransformer?: CatalogCollatorEntityTransformer;
};
/** @public */
export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
public readonly type: string = 'software-catalog';
public readonly type = 'software-catalog';
public readonly visibilityPermission: Permission =
catalogEntityReadPermission;
@@ -55,6 +57,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
private batchSize: number;
private readonly catalogClient: CatalogApi;
private tokenManager: TokenManager;
private entityTransformer: CatalogCollatorEntityTransformer;
static fromConfig(
_config: Config,
@@ -71,6 +74,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
filter,
catalogClient,
tokenManager,
entityTransformer,
} = options;
this.locationTemplate =
@@ -80,23 +84,14 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
this.catalogClient =
catalogClient || new CatalogClient({ discoveryApi: discovery });
this.tokenManager = tokenManager;
this.entityTransformer =
entityTransformer ?? defaultCatalogCollatorEntityTransformer;
}
async getCollator(): Promise<Readable> {
return Readable.from(this.execute());
}
private applyArgsToFormat(
format: string,
args: Record<string, string>,
): string {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted.toLowerCase();
}
private async *execute(): AsyncGenerator<CatalogEntityDocument> {
const { token } = await this.tokenManager.getToken();
let entitiesRetrieved = 0;
@@ -123,24 +118,30 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
for (const entity of entities) {
yield {
title: entity.metadata.title ?? entity.metadata.name,
...this.entityTransformer(entity),
authorization: {
resourceRef: stringifyEntityRef(entity),
},
location: this.applyArgsToFormat(this.locationTemplate, {
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
name: entity.metadata.name,
}),
text: getDocumentText(entity),
componentType: entity.spec?.type?.toString() || 'other',
type: entity.spec?.type?.toString() || 'other',
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: (entity.spec?.owner as string) || '',
authorization: {
resourceRef: stringifyEntityRef(entity),
},
};
}
}
}
private applyArgsToFormat(
format: string,
args: Record<string, string>,
): string {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted.toLowerCase();
}
}
plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts
+150
View File
@@ -0,0 +1,150 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
title: 'Test Entity',
name: 'test-entity',
description: 'The expected description',
namespace: 'namespace',
},
spec: {
type: 'some-type',
lifecycle: 'experimental',
owner: 'someone',
},
};
const userEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'test-user-entity',
description: 'The expected user description',
},
spec: {
profile: {
displayName: 'User 1',
},
},
};
describe('DefaultCatalogCollatorEntityTransformer', () => {
describe('transform', () => {
it('maps a returned entity', async () => {
const document = defaultCatalogCollatorEntityTransformer(entity);
expect(document).toMatchObject({
title: entity.metadata.title,
text: entity.metadata.description,
namespace: entity.metadata.namespace,
componentType: entity.spec.type,
lifecycle: entity.spec.lifecycle,
owner: entity.spec.owner,
});
});
it('maps a returned entity with default fallback', async () => {
const entityWithoutTitle = {
...entity,
metadata: {
...entity.metadata,
title: undefined,
namespace: undefined,
},
spec: {
type: undefined,
lifecycle: undefined,
owner: undefined,
},
};
const document =
defaultCatalogCollatorEntityTransformer(entityWithoutTitle);
expect(document).toMatchObject({
title: entity.metadata.name,
text: entity.metadata.description,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned user entity', async () => {
const document = defaultCatalogCollatorEntityTransformer(userEntity);
expect(document).toMatchObject({
title: userEntity.metadata.name,
text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned user entity without display name', async () => {
const testEntity = {
...userEntity,
spec: undefined,
};
const document = defaultCatalogCollatorEntityTransformer(testEntity);
expect(document).toMatchObject({
title: userEntity.metadata.name,
text: userEntity.metadata.description,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
it('maps a returned group entity', async () => {
const groupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: 'test-group-entity',
description: 'The expected group description',
},
spec: {
profile: {
displayName: 'Group 1',
},
},
};
const document = defaultCatalogCollatorEntityTransformer(groupEntity);
expect(document).toMatchObject({
title: groupEntity.metadata.name,
text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`,
namespace: 'default',
componentType: 'other',
lifecycle: '',
owner: '',
});
});
});
});
plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model';
import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
const getDocumentText = (entity: Entity): string => {
const documentTexts: string[] = [];
documentTexts.push(entity.metadata.description || '');
if (isUserEntity(entity) || isGroupEntity(entity)) {
if (entity.spec?.profile?.displayName) {
documentTexts.push(entity.spec.profile.displayName);
}
}
return documentTexts.join(' : ');
};
/** @public */
export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer =
(entity: Entity) => {
return {
title: entity.metadata.title ?? entity.metadata.name,
text: getDocumentText(entity),
componentType: entity.spec?.type?.toString() || 'other',
type: entity.spec?.type?.toString() || 'other',
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: (entity.spec?.owner as string) || '',
};
};
plugins/catalog-backend/src/search/index.ts
+2
View File
@@ -16,6 +16,8 @@
export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory';
export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory';
export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
export type { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer';
/**
* todo(backstage/techdocs-core): stop exporting this in a future release.
plugins/catalog-backend/src/search/util.test.ts
-146
View File
@@ -1,146 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
ComponentEntity,
GroupEntity,
UserEntity,
} from '@backstage/catalog-model';
import { getDocumentText } from './util';
describe('getDocumentText', () => {
describe('kind is not User or Group', () => {
test('contains description if set', () => {
const entity = createComponent();
entity.metadata.description = 'The expected description';
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description is not set', () => {
const entity = createComponent();
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
describe('kind is User', () => {
test('contains display name if set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
});
test('contains description if set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('contains both description and display name if both are set', () => {
const entity = createUser();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description and display name are not set', () => {
const entity = createUser();
delete entity.metadata.description;
delete entity.spec.profile?.displayName;
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
describe('kind is Group', () => {
test('contains display name if set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
});
test('contains description if set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.metadata.description);
});
test('contains both description and display name if both are set', () => {
const entity = createGroup();
const actual = getDocumentText(entity);
expect(actual).toContain(entity.spec.profile?.displayName);
expect(actual).toContain(entity.metadata.description);
});
test('is empty if description and display name are not set', () => {
const entity = createGroup();
delete entity.metadata.description;
delete entity.spec.profile?.displayName;
const actual = getDocumentText(entity);
expect(actual).toEqual('');
});
});
});
function createGroup(): GroupEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: 'group-1',
description: 'The expected description',
},
spec: {
type: 'team',
profile: {
displayName: 'Group 1',
},
children: [],
},
};
}
function createUser(): UserEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'user-1',
description: 'The expected description',
},
spec: {
profile: {
displayName: 'User 1',
},
},
};
}
function createComponent(): ComponentEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'component-1',
},
spec: {
lifecycle: 'experimental',
owner: 'someone',
type: 'service',
},
};
}