From f7bdceae07bf08f9ac80e8285063bb10db0e81bd Mon Sep 17 00:00:00 2001 From: Adam Wallis Date: Wed, 29 May 2024 16:59:04 -0400 Subject: [PATCH] Adds ProviderTransformer to msgraph catalog plugin for dynamic config Signed-off-by: Adam Wallis --- .changeset/clever-geese-jump.md | 5 +++ docs/integrations/azure/org.md | 34 ++++++++++++++++++- .../api-report-alpha.md | 6 ++++ .../api-report.md | 10 ++++++ .../src/microsoftGraph/index.ts | 1 + .../src/microsoftGraph/types.ts | 14 +++++++- ...ogModuleMicrosoftGraphOrgEntityProvider.ts | 24 +++++++++++++ .../MicrosoftGraphOrgEntityProvider.ts | 23 ++++++++++++- 8 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 .changeset/clever-geese-jump.md diff --git a/.changeset/clever-geese-jump.md b/.changeset/clever-geese-jump.md new file mode 100644 index 0000000000..1c4694fa33 --- /dev/null +++ b/.changeset/clever-geese-jump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Adds a dynamic provider for the plugin-catalog-backend-module-msgraph. Configuration is now runtime configurable through the ProviderConfigTransformer. diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 89abd4813b..6a72527b94 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -169,6 +169,23 @@ microsoftGraphOrg: select: ['id', 'displayName', 'description'] ``` +### Using Provider Config Transformer + +Dynamic configuration scaling allows the `msgraph` catalog plugin to adjust its settings at runtime without requiring a redeploy. This feature is useful for scenarios where configuration needs to be updated based on real-time events or changing conditions. For example, you can dynamically adjust synchronization schedules, filters, and search parameters to optimize performance and responsiveness. + +:::note +Adjusting fields that are not used on each scheduled ingestion (e.g., `id`, `schedule`) will have no effect. +::: + +:::warning +Dynamically changing configuration on the fly can introduce unintended consequences, such as system instability and configuration errors. Please review your transformer carefully to ensure that it is working as anticipated! +::: + +#### Example Use Cases: + +- **Filter Scaling**: Adjust filters like `userGroupMember` and `groupFilter` dynamically. +- **Search Parameter Adjustment**: Change search parameters such as `groupSearch` and `userSelect` on-the-fly. + ### Using Custom Transformers Transformers can be configured by extending `microsoftGraphOrgEntityProviderTransformExtensionPoint`. Here is an example: @@ -180,6 +197,7 @@ import { myUserTransformer, myGroupTransformer, myOrganizationTransformer, + myProviderConfigTransformer, } from './transformers'; backend.add( @@ -201,6 +219,9 @@ backend.add( microsoftGraphTransformers.setOrganizationTransformer( myOrganizationTransformer, ); + microsoftGraphTransformers.setProviderConfigTransformer( + myProviderConfigTransformer, + ); /* highlight-add-end */ }, }); @@ -209,7 +230,7 @@ backend.add( ); ``` -The `myUserTransformer`, `myGroupTransformer`, and `myOrganizationTransformer` transformer functions are from the examples in the section below. +The `myUserTransformer`, `myGroupTransformer`, `myOrganizationTransformer`, and `myProviderConfigTransformer` transformer functions are from the examples in the section below. ### Transformer Examples @@ -221,6 +242,7 @@ import { defaultGroupTransformer, defaultUserTransformer, defaultOrganizationTransformer, + MicrosoftGraphProviderConfig, } from '@backstage/plugin-catalog-backend-module-msgraph'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; @@ -263,6 +285,16 @@ export async function myOrganizationTransformer( ): Promise { return undefined; } + +// Example config transformer that expands the group filter to also include 'azure-group-a' +export async function myProviderConfigTransformer( + provider: MicrosoftGraphProviderConfig, +): Promise { + if (!provider.groupFilter?.includes('azure-group-a')) { + provider.groupFilter = `${provider.groupFilter} or displayName eq 'azure-group-a'`; + } + return provider; +} ``` ## Troubleshooting diff --git a/plugins/catalog-backend-module-msgraph/api-report-alpha.md b/plugins/catalog-backend-module-msgraph/api-report-alpha.md index b9c96e7798..7b0ced8625 100644 --- a/plugins/catalog-backend-module-msgraph/api-report-alpha.md +++ b/plugins/catalog-backend-module-msgraph/api-report-alpha.md @@ -7,6 +7,7 @@ import { BackendFeature } from '@backstage/backend-plugin-api'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; // @alpha @@ -26,6 +27,11 @@ export interface MicrosoftGraphOrgEntityProviderTransformsExtensionPoint { | OrganizationTransformer | Record, ): void; + setProviderConfigTransformer( + transformer: + | ProviderConfigTransformer + | Record, + ): void; setUserTransformer( transformer: UserTransformer | Record, ): void; diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 86170e2910..9640f3708b 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -126,6 +126,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -145,6 +146,7 @@ export interface MicrosoftGraphOrgEntityProviderLegacyOptions { id: string; logger: LoggerService; organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; schedule: 'manual' | TaskRunner; target: string; userTransformer?: UserTransformer; @@ -162,6 +164,9 @@ export type MicrosoftGraphOrgEntityProviderOptions = organizationTransformer?: | OrganizationTransformer | Record; + providerConfigTransformer?: + | ProviderConfigTransformer + | Record; }; // @public @deprecated @@ -233,6 +238,11 @@ export type OrganizationTransformer = ( organization: MicrosoftGraph.Organization, ) => Promise; +// @public +export type ProviderConfigTransformer = ( + provider: MicrosoftGraphProviderConfig, +) => Promise; + // @public @deprecated export function readMicrosoftGraphConfig( config: Config, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts index 4db3fb69c6..81f6f21ce2 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts @@ -39,4 +39,5 @@ export type { GroupTransformer, OrganizationTransformer, UserTransformer, + ProviderConfigTransformer, } from './types'; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts index ef60d7b018..bc9270238b 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts @@ -16,7 +16,7 @@ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; - +import { MicrosoftGraphProviderConfig } from './config'; /** * Customize the ingested User entity * @@ -45,3 +45,15 @@ export type GroupTransformer = ( group: MicrosoftGraph.Group, groupPhoto?: string, ) => Promise; + +/** + * Customize the MSGraph Provider Config Dynamically + * + * Transforming fields that are not used for each scheduled ingestion (e.g., id, schedule) + * will have no effect. + * + * @public + */ +export type ProviderConfigTransformer = ( + provider: MicrosoftGraphProviderConfig, +) => Promise; diff --git a/plugins/catalog-backend-module-msgraph/src/module/catalogModuleMicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/module/catalogModuleMicrosoftGraphOrgEntityProvider.ts index 193f07889f..e1a15a3468 100644 --- a/plugins/catalog-backend-module-msgraph/src/module/catalogModuleMicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/module/catalogModuleMicrosoftGraphOrgEntityProvider.ts @@ -23,6 +23,7 @@ import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/ import { GroupTransformer, OrganizationTransformer, + ProviderConfigTransformer, UserTransformer, } from '@backstage/plugin-catalog-backend-module-msgraph'; import { MicrosoftGraphOrgEntityProvider } from '../processors'; @@ -58,6 +59,18 @@ export interface MicrosoftGraphOrgEntityProviderTransformsExtensionPoint { | OrganizationTransformer | Record, ): void; + + /** + * Set the function that transforms provider config dynamically. + * Optionally, you can pass separate transformers per provider ID. + * Note: adjusting fields that are not used on each scheduled ingestion + * (e.g., id, schedule) will have no effect. + */ + setProviderConfigTransformer( + transformer: + | ProviderConfigTransformer + | Record, + ): void; } /** @@ -94,6 +107,10 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule( | OrganizationTransformer | Record | undefined; + let providerConfigTransformer: + | ProviderConfigTransformer + | Record + | undefined; env.registerExtensionPoint( microsoftGraphOrgEntityProviderTransformExtensionPoint, @@ -116,6 +133,12 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule( } organizationTransformer = transformer; }, + setProviderConfigTransformer(transformer) { + if (providerConfigTransformer) { + throw new Error('Provider transformer may only be set once'); + } + providerConfigTransformer = transformer; + }, }, ); @@ -134,6 +157,7 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule( userTransformer: userTransformer, groupTransformer: groupTransformer, organizationTransformer: organizationTransformer, + providerConfigTransformer: providerConfigTransformer, }), ); }, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 3c8b903e69..21995d2b8d 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -34,6 +34,7 @@ import { MICROSOFT_GRAPH_USER_ID_ANNOTATION, MicrosoftGraphClient, MicrosoftGraphProviderConfig, + ProviderConfigTransformer, OrganizationTransformer, readMicrosoftGraphConfig, readMicrosoftGraphOrg, @@ -94,6 +95,13 @@ export type MicrosoftGraphOrgEntityProviderOptions = organizationTransformer?: | OrganizationTransformer | Record; + + /** + * The function that transforms provider config dynamically. + */ + providerConfigTransformer?: + | ProviderConfigTransformer + | Record; }; /** @@ -152,6 +160,11 @@ export interface MicrosoftGraphOrgEntityProviderLegacyOptions { * The function that transforms an organization entry in msgraph to an entity. */ organizationTransformer?: OrganizationTransformer; + + /** + * The function that transforms provider config dynamically. + */ + providerConfigTransformer?: ProviderConfigTransformer; } /** @@ -216,6 +229,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { providerConfig.id, options.organizationTransformer, ), + providerConfigTransformer: getTransformer( + providerConfig.id, + options.providerConfigTransformer, + ), }); if (taskRunner !== 'manual') { @@ -257,6 +274,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { userTransformer: options.userTransformer, groupTransformer: options.groupTransformer, organizationTransformer: options.organizationTransformer, + providerConfigTransformer: options.providerConfigTransformer, logger, provider, }); @@ -276,6 +294,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; }, ) {} @@ -300,7 +319,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { } const logger = options?.logger ?? this.options.logger; - const provider = this.options.provider; + const provider = this.options.providerConfigTransformer + ? await this.options.providerConfigTransformer(this.options.provider) + : this.options.provider; const { markReadComplete } = trackProgress(logger); const client = MicrosoftGraphClient.create(this.options.provider); const { users, groups } = await readMicrosoftGraphOrg(