paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

fix(catalog-backend): let processors validate kinds (#3113)

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-10-28 16:04:34 +01:00
committed by GitHub
parent 183e2a30de
commit 5adfc005e8
21 changed files with 289 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+23
View File
@@ -211,6 +211,29 @@ export class LocationReaders implements LocationReader {
return undefined;
}
let handled = false;
for (const processor of processors) {
if (processor.validateEntityKind) {
try {
handled = await processor.validateEntityKind(current);
if (handled) {
break;
}
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while validating the entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
if (!handled) {
const message = `No processor recognized the entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
for (const processor of processors) {
if (processor.postProcessEntity) {
try {
plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
apiEntityV1alpha1Validator,
componentEntityV1alpha1Validator,
Entity,
groupEntityV1alpha1Validator,
locationEntityV1alpha1Validator,
templateEntityV1alpha1Validator,
userEntityV1alpha1Validator,
} from '@backstage/catalog-model';
import { CatalogProcessor } from './types';
export class BuiltinKindsEntityProcessor implements CatalogProcessor {
private readonly validators = [
apiEntityV1alpha1Validator,
componentEntityV1alpha1Validator,
groupEntityV1alpha1Validator,
locationEntityV1alpha1Validator,
templateEntityV1alpha1Validator,
userEntityV1alpha1Validator,
];
async validateEntityKind(entity: Entity): Promise<boolean> {
for (const validator of this.validators) {
const result = await validator.check(entity);
if (result) {
return true;
}
}
return false;
}
}
plugins/catalog-backend/src/ingestion/processors/types.ts
+13
View File
@@ -54,6 +54,19 @@ export type CatalogProcessor = {
emit: CatalogProcessorEmit,
): Promise<Entity>;
/**
* Validates the entity as a known entity kind, after it has been pre-
* processed and has passed through basic overall validation.
*
* @param entity The entity to validate
* @returns Resolves to true, if the entity was of a kind that was known and
* handled by this processor, and was found to be valid. Resolves to false,
* if the entity was not of a kind that was known by this processor.
* Rejects to an Error describing the problem, if the entity was of a kind
* that was known by this processor and was not valid.
*/
validateEntityKind?(entity: Entity): Promise<boolean>;
/**
* Post-processes an emitted entity, after it has been validated.
*
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+10 -12
View File
@@ -76,14 +76,6 @@ describe('CatalogBuilder', () => {
},
},
])
.replaceEntityKinds([
{
async enforce(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return entity;
},
},
])
.setPlaceholderResolver('t', async ({ value }) => {
expect(value).toBe('tt');
return 'tt2';
@@ -100,15 +92,16 @@ describe('CatalogBuilder', () => {
expect(location.type).toBe('test');
emit(
result.entity(location, {
apiVersion: 'av',
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: { name: 'n', replaced: { $t: 'tt' } },
spec: { type: 't', owner: 'o', lifecycle: 'l' },
}),
);
return true;
},
async preProcessEntity(entity) {
expect(entity.apiVersion).toBe('av');
expect(entity.apiVersion).toBe('backstage.io/v1alpha1');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
@@ -129,10 +122,10 @@ describe('CatalogBuilder', () => {
type: 'test',
target: '',
});
expect.assertions(7);
expect.assertions(6);
expect(added.entities).toEqual([
{
apiVersion: 'av',
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'n',
@@ -143,6 +136,11 @@ describe('CatalogBuilder', () => {
etag: expect.any(String),
generation: expect.any(Number),
},
spec: {
type: 't',
owner: 'o',
lifecycle: 'l',
},
relations: [],
},
]);
plugins/catalog-backend/src/service/CatalogBuilder.ts
+6 -61
View File
@@ -16,19 +16,13 @@
import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common';
import {
apiEntityV1alpha1Policy,
componentEntityV1alpha1Policy,
DefaultNamespaceEntityPolicy,
EntityPolicies,
EntityPolicy,
FieldFormatEntityPolicy,
groupEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
makeValidator,
NoForeignRootFieldsEntityPolicy,
SchemaValidEntityPolicy,
templateEntityV1alpha1Policy,
userEntityV1alpha1Policy,
Validators,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
@@ -47,17 +41,18 @@ import {
CodeOwnersProcessor,
FileReaderProcessor,
GithubOrgReaderProcessor,
OwnerRelationProcessor,
HigherOrderOperation,
HigherOrderOperations,
LocationReaders,
LocationRefProcessor,
OwnerRelationProcessor,
PlaceholderProcessor,
PlaceholderResolver,
StaticLocationProcessor,
UrlReaderProcessor,
} from '../ingestion';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { BuiltinKindsEntityProcessor } from '../ingestion/processors/BuiltinKindsEntityProcessor';
import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor';
import {
jsonPlaceholderResolver,
@@ -81,11 +76,6 @@ export type CatalogEnvironment = {
* after the processors' pre-processing steps. All policies are given the
* chance to inspect the entity, and all of them have to pass in order for
* the entity to be considered valid from an overall point of view.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which adds
* additional kind-specific validation (usually based on a schema). Only one
* of the entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Placeholder resolvers can be replaced or added. These run on the raw
* structured data between the parsing and pre-processing steps, to replace
* dollar-prefixed entries with their actual values (like $file).
@@ -93,15 +83,13 @@ export type CatalogEnvironment = {
* individual core fields such as metadata.name, to ensure that they adhere
* to certain rules.
* - Processors can be added or replaced. These implement the functionality of
* reading, parsing and processing the entity data before it is persisted in
* the catalog.
* reading, parsing, validating, and processing the entity data before it is
* persisted in the catalog.
*/
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
private entityPolicies: EntityPolicy[];
private entityPoliciesReplace: boolean;
private entityKinds: EntityPolicy[];
private entityKindsReplace: boolean;
private placeholderResolvers: Record<string, PlaceholderResolver>;
private fieldFormatValidators: Partial<Validators>;
private processors: CatalogProcessor[];
@@ -111,8 +99,6 @@ export class CatalogBuilder {
this.env = env;
this.entityPolicies = [];
this.entityPoliciesReplace = false;
this.entityKinds = [];
this.entityKindsReplace = false;
this.placeholderResolvers = {};
this.fieldFormatValidators = {};
this.processors = [];
@@ -154,33 +140,6 @@ export class CatalogBuilder {
return this;
}
/**
* Adds entity kinds that are used to validate a certain apiVersion/kind. One
* of the entity kind policies must match a given entity for it to be
* considered valid.
*
* @param policies One or more policies
*/
addEntityKind(...policies: EntityPolicy[]): CatalogBuilder {
this.entityKinds.push(...policies);
return this;
}
/**
* Sets what entity policies that are used to validate a certain apiVersion/
* kind. One of the entity kind policies must match a given entity for it to
* be considered valid.
*
* This function replaces the default set of kinds; use with care.
*
* @param policies One or more policies
*/
replaceEntityKinds(policies: EntityPolicy[]): CatalogBuilder {
this.entityKinds = [...policies];
this.entityKindsReplace = true;
return this;
}
/**
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
* definition files.
@@ -291,22 +250,7 @@ export class CatalogBuilder {
...this.entityPolicies,
];
const entityKinds: EntityPolicy[] = this.entityKindsReplace
? this.entityKinds
: [
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
userEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
apiEntityV1alpha1Policy,
...this.entityKinds,
];
return EntityPolicies.allOf([
EntityPolicies.allOf(entityPolicies),
EntityPolicies.oneOf(entityKinds),
]);
return EntityPolicies.allOf(entityPolicies);
}
private buildProcessors(): CatalogProcessor[] {
@@ -325,6 +269,7 @@ export class CatalogBuilder {
const processors: CatalogProcessor[] = [
StaticLocationProcessor.fromConfig(config),
new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }),
new BuiltinKindsEntityProcessor(),
];
// These are only added unless the user replaced them all