Adds ProviderTransformer to msgraph catalog plugin for dynamic config

Signed-off-by: Adam Wallis <adam.wallis@gmail.com>

.changeset/clever-geese-jump.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-msgraph': patch
+---
+
+Adds a dynamic provider for the plugin-catalog-backend-module-msgraph. Configuration is now runtime configurable through the ProviderConfigTransformer.

docs/integrations/azure/org.md

@@ -169,6 +169,23 @@ microsoftGraphOrg:
       select: ['id', 'displayName', 'description']
 ```
 
+### Using Provider Config Transformer
+
+Dynamic configuration scaling allows the `msgraph` catalog plugin to adjust its settings at runtime without requiring a redeploy. This feature is useful for scenarios where configuration needs to be updated based on real-time events or changing conditions. For example, you can dynamically adjust synchronization schedules, filters, and search parameters to optimize performance and responsiveness.
+
+:::note
+Adjusting fields that are not used on each scheduled ingestion (e.g., `id`, `schedule`) will have no effect.
+:::
+
+:::warning
+Dynamically changing configuration on the fly can introduce unintended consequences, such as system instability and configuration errors. Please review your transformer carefully to ensure that it is working as anticipated!
+:::
+
+#### Example Use Cases:
+
+- **Filter Scaling**: Adjust filters like `userGroupMember` and `groupFilter` dynamically.
+- **Search Parameter Adjustment**: Change search parameters such as `groupSearch` and `userSelect` on-the-fly.
+
 ### Using Custom Transformers
 
 Transformers can be configured by extending `microsoftGraphOrgEntityProviderTransformExtensionPoint`. Here is an example:
@@ -180,6 +197,7 @@ import {
   myUserTransformer,
   myGroupTransformer,
   myOrganizationTransformer,
+  myProviderConfigTransformer,
 } from './transformers';
 
 backend.add(
@@ -201,6 +219,9 @@ backend.add(
           microsoftGraphTransformers.setOrganizationTransformer(
             myOrganizationTransformer,
           );
+          microsoftGraphTransformers.setProviderConfigTransformer(
+            myProviderConfigTransformer,
+          );
           /* highlight-add-end */
         },
       });
@@ -209,7 +230,7 @@ backend.add(
 );
 ```
 
-The `myUserTransformer`, `myGroupTransformer`, and `myOrganizationTransformer` transformer functions are from the examples in the section below.
+The `myUserTransformer`, `myGroupTransformer`, `myOrganizationTransformer`, and `myProviderConfigTransformer` transformer functions are from the examples in the section below.
 
 ### Transformer Examples
 
@@ -221,6 +242,7 @@ import {
   defaultGroupTransformer,
   defaultUserTransformer,
   defaultOrganizationTransformer,
+  MicrosoftGraphProviderConfig,
 } from '@backstage/plugin-catalog-backend-module-msgraph';
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 
@@ -263,6 +285,16 @@ export async function myOrganizationTransformer(
 ): Promise<GroupEntity | undefined> {
   return undefined;
 }
+
+// Example config transformer that expands the group filter to also include 'azure-group-a'
+export async function myProviderConfigTransformer(
+  provider: MicrosoftGraphProviderConfig,
+): Promise<MicrosoftGraphProviderConfig> {
+  if (!provider.groupFilter?.includes('azure-group-a')) {
+    provider.groupFilter = `${provider.groupFilter} or displayName eq 'azure-group-a'`;
+  }
+  return provider;
+}
 ```
 
 ## Troubleshooting

plugins/catalog-backend-module-msgraph/api-report-alpha.md

@@ -7,6 +7,7 @@ import { BackendFeature } from '@backstage/backend-plugin-api';
 import { ExtensionPoint } from '@backstage/backend-plugin-api';
 import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
 import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
+import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
 import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
 
 // @alpha
@@ -26,6 +27,11 @@ export interface MicrosoftGraphOrgEntityProviderTransformsExtensionPoint {
       | OrganizationTransformer
       | Record<string, OrganizationTransformer>,
   ): void;
+  setProviderConfigTransformer(
+    transformer:
+      | ProviderConfigTransformer
+      | Record<string, ProviderConfigTransformer>,
+  ): void;
   setUserTransformer(
     transformer: UserTransformer | Record<string, UserTransformer>,
   ): void;

plugins/catalog-backend-module-msgraph/api-report.md

@@ -126,6 +126,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
     userTransformer?: UserTransformer;
     groupTransformer?: GroupTransformer;
     organizationTransformer?: OrganizationTransformer;
+    providerConfigTransformer?: ProviderConfigTransformer;
   });
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
@@ -145,6 +146,7 @@ export interface MicrosoftGraphOrgEntityProviderLegacyOptions {
   id: string;
   logger: LoggerService;
   organizationTransformer?: OrganizationTransformer;
+  providerConfigTransformer?: ProviderConfigTransformer;
   schedule: 'manual' | TaskRunner;
   target: string;
   userTransformer?: UserTransformer;
@@ -162,6 +164,9 @@ export type MicrosoftGraphOrgEntityProviderOptions =
       organizationTransformer?:
         | OrganizationTransformer
         | Record<string, OrganizationTransformer>;
+      providerConfigTransformer?:
+        | ProviderConfigTransformer
+        | Record<string, ProviderConfigTransformer>;
     };
 
 // @public @deprecated
@@ -233,6 +238,11 @@ export type OrganizationTransformer = (
   organization: MicrosoftGraph.Organization,
 ) => Promise<GroupEntity | undefined>;
 
+// @public
+export type ProviderConfigTransformer = (
+  provider: MicrosoftGraphProviderConfig,
+) => Promise<MicrosoftGraphProviderConfig>;
+
 // @public @deprecated
 export function readMicrosoftGraphConfig(
   config: Config,

plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts

@@ -39,4 +39,5 @@ export type {
   GroupTransformer,
   OrganizationTransformer,
   UserTransformer,
+  ProviderConfigTransformer,
 } from './types';

plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts

@@ -16,7 +16,7 @@
 
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
-
+import { MicrosoftGraphProviderConfig } from './config';
 /**
  * Customize the ingested User entity
  *
@@ -45,3 +45,15 @@ export type GroupTransformer = (
   group: MicrosoftGraph.Group,
   groupPhoto?: string,
 ) => Promise<GroupEntity | undefined>;
+
+/**
+ * Customize the MSGraph Provider Config Dynamically
+ *
+ * Transforming fields that are not used for each scheduled ingestion (e.g., id, schedule)
+ * will have no effect.
+ *
+ * @public
+ */
+export type ProviderConfigTransformer = (
+  provider: MicrosoftGraphProviderConfig,
+) => Promise<MicrosoftGraphProviderConfig>;

plugins/catalog-backend-module-msgraph/src/module/catalogModuleMicrosoftGraphOrgEntityProvider.ts

@@ -23,6 +23,7 @@ import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/
 import {
   GroupTransformer,
   OrganizationTransformer,
+  ProviderConfigTransformer,
   UserTransformer,
 } from '@backstage/plugin-catalog-backend-module-msgraph';
 import { MicrosoftGraphOrgEntityProvider } from '../processors';
@@ -58,6 +59,18 @@ export interface MicrosoftGraphOrgEntityProviderTransformsExtensionPoint {
       | OrganizationTransformer
       | Record<string, OrganizationTransformer>,
   ): void;
+
+  /**
+   * Set the function that transforms provider config dynamically.
+   * Optionally, you can pass separate transformers per provider ID.
+   * Note: adjusting fields that are not used on each scheduled ingestion
+   *       (e.g., id, schedule) will have no effect.
+   */
+  setProviderConfigTransformer(
+    transformer:
+      | ProviderConfigTransformer
+      | Record<string, ProviderConfigTransformer>,
+  ): void;
 }
 
 /**
@@ -94,6 +107,10 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule(
         | OrganizationTransformer
         | Record<string, OrganizationTransformer>
         | undefined;
+      let providerConfigTransformer:
+        | ProviderConfigTransformer
+        | Record<string, ProviderConfigTransformer>
+        | undefined;
 
       env.registerExtensionPoint(
         microsoftGraphOrgEntityProviderTransformExtensionPoint,
@@ -116,6 +133,12 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule(
             }
             organizationTransformer = transformer;
           },
+          setProviderConfigTransformer(transformer) {
+            if (providerConfigTransformer) {
+              throw new Error('Provider transformer may only be set once');
+            }
+            providerConfigTransformer = transformer;
+          },
         },
       );
 
@@ -134,6 +157,7 @@ export const catalogModuleMicrosoftGraphOrgEntityProvider = createBackendModule(
               userTransformer: userTransformer,
               groupTransformer: groupTransformer,
               organizationTransformer: organizationTransformer,
+              providerConfigTransformer: providerConfigTransformer,
             }),
           );
         },

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts

@@ -34,6 +34,7 @@ import {
   MICROSOFT_GRAPH_USER_ID_ANNOTATION,
   MicrosoftGraphClient,
   MicrosoftGraphProviderConfig,
+  ProviderConfigTransformer,
   OrganizationTransformer,
   readMicrosoftGraphConfig,
   readMicrosoftGraphOrg,
@@ -94,6 +95,13 @@ export type MicrosoftGraphOrgEntityProviderOptions =
       organizationTransformer?:
         | OrganizationTransformer
         | Record<string, OrganizationTransformer>;
+
+      /**
+       * The function that transforms provider config dynamically.
+       */
+      providerConfigTransformer?:
+        | ProviderConfigTransformer
+        | Record<string, ProviderConfigTransformer>;
     };
 
 /**
@@ -152,6 +160,11 @@ export interface MicrosoftGraphOrgEntityProviderLegacyOptions {
    * The function that transforms an organization entry in msgraph to an entity.
    */
   organizationTransformer?: OrganizationTransformer;
+
+  /**
+   *  The function that transforms provider config dynamically.
+   */
+  providerConfigTransformer?: ProviderConfigTransformer;
 }
 
 /**
@@ -216,6 +229,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
           providerConfig.id,
           options.organizationTransformer,
         ),
+        providerConfigTransformer: getTransformer(
+          providerConfig.id,
+          options.providerConfigTransformer,
+        ),
       });
 
       if (taskRunner !== 'manual') {
@@ -257,6 +274,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
       userTransformer: options.userTransformer,
       groupTransformer: options.groupTransformer,
       organizationTransformer: options.organizationTransformer,
+      providerConfigTransformer: options.providerConfigTransformer,
       logger,
       provider,
     });
@@ -276,6 +294,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
       userTransformer?: UserTransformer;
       groupTransformer?: GroupTransformer;
       organizationTransformer?: OrganizationTransformer;
+      providerConfigTransformer?: ProviderConfigTransformer;
     },
   ) {}
 
@@ -300,7 +319,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
     }
 
     const logger = options?.logger ?? this.options.logger;
-    const provider = this.options.provider;
+    const provider = this.options.providerConfigTransformer
+      ? await this.options.providerConfigTransformer(this.options.provider)
+      : this.options.provider;
     const { markReadComplete } = trackProgress(logger);
     const client = MicrosoftGraphClient.create(this.options.provider);
     const { users, groups } = await readMicrosoftGraphOrg(