feat(catalog): add catalog-backend-module-msgraph-incremental plugin
Introduces a new Backstage backend module that incrementally ingests users and groups from Microsoft Graph one page at a time, using the incremental ingestion framework. Unlike MicrosoftGraphOrgEntityProvider, this module never holds the full dataset in memory. Each burst processes a single page (up to 999 items), making it suitable for large Azure AD tenants where the full-scan provider causes memory pressure or OOM failures. The @odata.nextLink cursor is persisted in the incremental ingestion marks table, so a pod restart during ingestion resumes from the last completed page rather than starting over. Signed-off-by: pillaris <pillaris@adobe.com>
This commit is contained in:
pillaris
@@ -0,0 +1,7 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-msgraph-incremental': patch
|
||||
---
|
||||
|
||||
New package: `@backstage/plugin-catalog-backend-module-msgraph-incremental`
|
||||
|
||||
Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page of up to 999 items. The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over.
|
||||
@@ -58,6 +58,7 @@ yarn.lock @backstage/maintainers @backst
|
||||
/plugins/catalog-backend-module-backstage-openapi @backstage/maintainers @backstage/openapi-tooling-maintainers
|
||||
/plugins/catalog-backend-module-gitea @backstage/maintainers @backstage/catalog-maintainers
|
||||
/plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-maintainers @pjungermann
|
||||
/plugins/catalog-backend-module-msgraph-incremental @backstage/maintainers @backstage/catalog-maintainers
|
||||
/plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-maintainers
|
||||
/plugins/catalog-graph @backstage/maintainers @backstage/catalog-maintainers @backstage/sda-se-reviewers
|
||||
/plugins/devtools @backstage/maintainers @awanlin
|
||||
|
||||
@@ -41,7 +41,10 @@
|
||||
"@backstage/plugin-auth-node": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-logs": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-openapi": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^",
|
||||
|
||||
@@ -46,6 +46,12 @@ backend.add(import('@backstage/plugin-auth-backend-module-guest-provider'));
|
||||
backend.add(import('@backstage/plugin-auth-backend-module-openshift-provider'));
|
||||
backend.add(import('@backstage/plugin-app-backend'));
|
||||
backend.add(import('@backstage/plugin-catalog-backend-module-unprocessed'));
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
|
||||
);
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-msgraph-incremental'),
|
||||
);
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'),
|
||||
);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
|
||||
@@ -0,0 +1,16 @@
|
||||
# @backstage/plugin-catalog-backend-module-msgraph-incremental
|
||||
|
||||
## 0.1.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Initial release of the incremental Microsoft Graph catalog module.
|
||||
|
||||
Provides `catalogModuleMicrosoftGraphIncrementalEntityProvider`, which
|
||||
ingests users and groups from Microsoft Graph one page at a time using the
|
||||
catalog's incremental ingestion framework. Unlike
|
||||
`MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset
|
||||
in memory and resumes from its cursor on pod restart.
|
||||
|
||||
Requires `@backstage/plugin-catalog-backend-module-incremental-ingestion` to
|
||||
be installed in the backend.
|
||||
@@ -0,0 +1,73 @@
|
||||
# @backstage/plugin-catalog-backend-module-msgraph-incremental
|
||||
|
||||
This module incrementally ingests **users** and **groups** from Microsoft Graph
|
||||
into the Backstage catalog, one page at a time. It is suitable for large Azure
|
||||
AD tenants where holding the full dataset in memory at once is not practical.
|
||||
|
||||
## Features
|
||||
|
||||
- **Cursor-based resumption** — the `@odata.nextLink` URL is persisted as the
|
||||
cursor, so a pod restart during ingestion resumes from the last completed page
|
||||
rather than starting over.
|
||||
- **Memory-efficient** — each burst processes a single page (up to 999 items),
|
||||
keeping memory usage flat regardless of tenant size.
|
||||
- **Photo support** — user profile photos are fetched with a gated pre-check to
|
||||
avoid unnecessary API calls for users without photos.
|
||||
- **Transformer extension point** — user, group, organization, and provider
|
||||
config transformers can be customised via the
|
||||
`microsoftGraphIncrementalEntityProviderTransformExtensionPoint`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
This module requires the incremental ingestion framework to be installed:
|
||||
|
||||
```ts
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
|
||||
);
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```ts
|
||||
// packages/backend/src/index.ts
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
|
||||
);
|
||||
backend.add(
|
||||
import('@backstage/plugin-catalog-backend-module-msgraph-incremental'),
|
||||
);
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Uses the same `catalog.providers.microsoftGraphOrg` configuration as
|
||||
`@backstage/plugin-catalog-backend-module-msgraph`. See that package's
|
||||
documentation for full config reference.
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
microsoftGraphOrg:
|
||||
default:
|
||||
tenantId: ${AZURE_TENANT_ID}
|
||||
clientId: ${AZURE_CLIENT_ID}
|
||||
clientSecret: ${AZURE_CLIENT_SECRET}
|
||||
queryMode: advanced
|
||||
user:
|
||||
filter: 'accountEnabled eq true'
|
||||
group:
|
||||
filter: 'securityEnabled eq true'
|
||||
schedule:
|
||||
frequency: { hours: 12 }
|
||||
timeout: { hours: 4 }
|
||||
```
|
||||
|
||||
## Differences from `MicrosoftGraphOrgEntityProvider`
|
||||
|
||||
| | `MicrosoftGraphOrgEntityProvider` | This module |
|
||||
| -------------------------- | --------------------------------- | ------------------- |
|
||||
| Memory usage | Full dataset in RAM | One page at a time |
|
||||
| Resume on restart | Starts from scratch | Resumes from cursor |
|
||||
| `userGroupMember*` options | Supported | Not supported |
|
||||
| Suitable for large tenants | No | Yes |
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: backstage.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: backstage-plugin-catalog-backend-module-msgraph-incremental
|
||||
title: '@backstage/plugin-catalog-backend-module-msgraph-incremental'
|
||||
description: >-
|
||||
A Backstage catalog backend module that incrementally ingests users and
|
||||
groups from Microsoft Graph, one page at a time
|
||||
spec:
|
||||
lifecycle: experimental
|
||||
type: backstage-backend-plugin-module
|
||||
owner: catalog-maintainers
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"name": "@backstage/plugin-catalog-backend-module-msgraph-incremental",
|
||||
"version": "0.1.0",
|
||||
"description": "A Backstage catalog backend module that incrementally ingests users and groups from Microsoft Graph",
|
||||
"backstage": {
|
||||
"role": "backend-plugin-module",
|
||||
"pluginId": "catalog",
|
||||
"pluginPackage": "@backstage/plugin-catalog-backend"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"keywords": [
|
||||
"backstage"
|
||||
],
|
||||
"homepage": "https://backstage.io",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/backstage/backstage",
|
||||
"directory": "plugins/catalog-backend-module-msgraph-incremental"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./alpha": "./src/alpha.ts",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"typesVersions": {
|
||||
"*": {
|
||||
"alpha": [
|
||||
"src/alpha.ts"
|
||||
],
|
||||
"package.json": [
|
||||
"package.json"
|
||||
]
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "backstage-cli package build",
|
||||
"clean": "backstage-cli package clean",
|
||||
"lint": "backstage-cli package lint",
|
||||
"prepack": "backstage-cli package prepack",
|
||||
"postpack": "backstage-cli package postpack",
|
||||
"start": "backstage-cli package start",
|
||||
"test": "backstage-cli package test"
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/backend-plugin-api": "workspace:^",
|
||||
"@backstage/catalog-model": "workspace:^",
|
||||
"@backstage/config": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^",
|
||||
"@backstage/plugin-catalog-node": "workspace:^",
|
||||
"@backstage/types": "workspace:^",
|
||||
"@microsoft/microsoft-graph-types": "^2.6.0",
|
||||
"p-limit": "^3.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "workspace:^"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { BackendFeature } from '@backstage/backend-plugin-api';
|
||||
import { Config } from '@backstage/config';
|
||||
import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { ExtensionPoint } from '@backstage/backend-plugin-api';
|
||||
import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
|
||||
// @public
|
||||
const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature;
|
||||
export { catalogModuleMicrosoftGraphIncrementalEntityProvider };
|
||||
export default catalogModuleMicrosoftGraphIncrementalEntityProvider;
|
||||
|
||||
// @public
|
||||
export class MicrosoftGraphIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<MSGraphCursor, MSGraphContext>
|
||||
{
|
||||
constructor(options: {
|
||||
id: string;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
logger: LoggerService;
|
||||
userTransformer?: UserTransformer;
|
||||
groupTransformer?: GroupTransformer;
|
||||
organizationTransformer?: OrganizationTransformer;
|
||||
providerConfigTransformer?: ProviderConfigTransformer;
|
||||
});
|
||||
around(burst: (context: MSGraphContext) => Promise<void>): Promise<void>;
|
||||
static fromConfig(
|
||||
configRoot: Config,
|
||||
options: MicrosoftGraphIncrementalEntityProviderOptions,
|
||||
): MicrosoftGraphIncrementalEntityProvider[];
|
||||
getProviderName(): string;
|
||||
next(
|
||||
input: MSGraphContext,
|
||||
cursor?: MSGraphCursor,
|
||||
): Promise<EntityIteratorResult<MSGraphCursor>>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface MicrosoftGraphIncrementalEntityProviderOptions {
|
||||
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
|
||||
logger: LoggerService;
|
||||
organizationTransformer?:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>;
|
||||
providerConfigTransformer?:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>;
|
||||
userTransformer?: UserTransformer | Record<string, UserTransformer>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint<MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint>;
|
||||
|
||||
// @public
|
||||
export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint {
|
||||
setGroupTransformer(
|
||||
transformer: GroupTransformer | Record<string, GroupTransformer>,
|
||||
): void;
|
||||
setOrganizationTransformer(
|
||||
transformer:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>,
|
||||
): void;
|
||||
setProviderConfigTransformer(
|
||||
transformer:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>,
|
||||
): void;
|
||||
setUserTransformer(
|
||||
transformer: UserTransformer | Record<string, UserTransformer>,
|
||||
): void;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type MSGraphContext = {
|
||||
client: MicrosoftGraphClient;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type MSGraphCursor = {
|
||||
phase: 'users' | 'groups';
|
||||
nextLink: string | null;
|
||||
};
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
```
|
||||
@@ -0,0 +1,96 @@
|
||||
## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { BackendFeature } from '@backstage/backend-plugin-api';
|
||||
import { Config } from '@backstage/config';
|
||||
import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { ExtensionPoint } from '@backstage/backend-plugin-api';
|
||||
import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
|
||||
// @public
|
||||
const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature;
|
||||
export { catalogModuleMicrosoftGraphIncrementalEntityProvider };
|
||||
export default catalogModuleMicrosoftGraphIncrementalEntityProvider;
|
||||
|
||||
// @public
|
||||
export class MicrosoftGraphIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<MSGraphCursor, MSGraphContext>
|
||||
{
|
||||
constructor(options: {
|
||||
id: string;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
logger: LoggerService;
|
||||
userTransformer?: UserTransformer;
|
||||
groupTransformer?: GroupTransformer;
|
||||
organizationTransformer?: OrganizationTransformer;
|
||||
providerConfigTransformer?: ProviderConfigTransformer;
|
||||
});
|
||||
around(burst: (context: MSGraphContext) => Promise<void>): Promise<void>;
|
||||
static fromConfig(
|
||||
configRoot: Config,
|
||||
options: MicrosoftGraphIncrementalEntityProviderOptions,
|
||||
): MicrosoftGraphIncrementalEntityProvider[];
|
||||
getProviderName(): string;
|
||||
next(
|
||||
input: MSGraphContext,
|
||||
cursor?: MSGraphCursor,
|
||||
): Promise<EntityIteratorResult<MSGraphCursor>>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface MicrosoftGraphIncrementalEntityProviderOptions {
|
||||
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
|
||||
logger: LoggerService;
|
||||
organizationTransformer?:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>;
|
||||
providerConfigTransformer?:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>;
|
||||
userTransformer?: UserTransformer | Record<string, UserTransformer>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint<MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint>;
|
||||
|
||||
// @public
|
||||
export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint {
|
||||
setGroupTransformer(
|
||||
transformer: GroupTransformer | Record<string, GroupTransformer>,
|
||||
): void;
|
||||
setOrganizationTransformer(
|
||||
transformer:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>,
|
||||
): void;
|
||||
setProviderConfigTransformer(
|
||||
transformer:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>,
|
||||
): void;
|
||||
setUserTransformer(
|
||||
transformer: UserTransformer | Record<string, UserTransformer>,
|
||||
): void;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type MSGraphContext = {
|
||||
client: MicrosoftGraphClient;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type MSGraphCursor = {
|
||||
phase: 'users' | 'groups';
|
||||
nextLink: string | null;
|
||||
};
|
||||
```
|
||||
+779
@@ -0,0 +1,779 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
} from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
MicrosoftGraphClient,
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
|
||||
MICROSOFT_GRAPH_USER_ID_ANNOTATION,
|
||||
} from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
import {
|
||||
MicrosoftGraphIncrementalEntityProvider,
|
||||
MSGraphContext,
|
||||
MSGraphCursor,
|
||||
} from './MicrosoftGraphIncrementalEntityProvider';
|
||||
import { getUserPhotoGated, requestOnePage } from './clientHelpers';
|
||||
|
||||
jest.mock('./clientHelpers', () => ({
|
||||
requestOnePage: jest.fn(),
|
||||
getUserPhotoGated: jest.fn(),
|
||||
}));
|
||||
|
||||
const mockRequestOnePage = requestOnePage as jest.MockedFunction<
|
||||
typeof requestOnePage
|
||||
>;
|
||||
const mockGetUserPhotoGated = getUserPhotoGated as jest.MockedFunction<
|
||||
typeof getUserPhotoGated
|
||||
>;
|
||||
|
||||
const mockClient = {
|
||||
getOrganization: jest.fn(),
|
||||
getGroupMembers: jest.fn(),
|
||||
} as unknown as jest.Mocked<MicrosoftGraphClient>;
|
||||
|
||||
const logger = mockServices.logger.mock();
|
||||
|
||||
const baseProviderConfig = {
|
||||
id: 'default',
|
||||
target: 'https://graph.microsoft.com/v1.0',
|
||||
tenantId: 'tenant-id',
|
||||
clientId: 'client-id',
|
||||
clientSecret: 'client-secret',
|
||||
};
|
||||
|
||||
function makeContext(overrides?: Partial<MSGraphContext>): MSGraphContext {
|
||||
return {
|
||||
client: mockClient,
|
||||
provider: baseProviderConfig as any,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function* asyncYield<T>(...items: T[]): AsyncIterable<T> {
|
||||
for (const item of items) yield item;
|
||||
}
|
||||
|
||||
describe('MicrosoftGraphIncrementalEntityProvider', () => {
|
||||
beforeEach(() => {
|
||||
jest
|
||||
.spyOn(MicrosoftGraphClient, 'create')
|
||||
.mockReturnValue(mockClient as unknown as MicrosoftGraphClient);
|
||||
});
|
||||
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
describe('getProviderName', () => {
|
||||
it('returns namespaced name using the provider id', () => {
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'my-tenant',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
expect(provider.getProviderName()).toBe(
|
||||
'MicrosoftGraphIncrementalEntityProvider:my-tenant',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fromConfig', () => {
|
||||
it('creates one provider per provider config entry', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
microsoftGraphOrg: {
|
||||
tenantA: { tenantId: 'a', clientId: 'c', clientSecret: 's' },
|
||||
tenantB: { tenantId: 'b', clientId: 'c', clientSecret: 's' },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const providers = MicrosoftGraphIncrementalEntityProvider.fromConfig(
|
||||
config,
|
||||
{ logger },
|
||||
);
|
||||
|
||||
expect(providers).toHaveLength(2);
|
||||
expect(providers[0].getProviderName()).toBe(
|
||||
'MicrosoftGraphIncrementalEntityProvider:tenantA',
|
||||
);
|
||||
expect(providers[1].getProviderName()).toBe(
|
||||
'MicrosoftGraphIncrementalEntityProvider:tenantB',
|
||||
);
|
||||
});
|
||||
|
||||
it('assigns per-provider transformers when a Record is provided', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
microsoftGraphOrg: {
|
||||
p1: { tenantId: 't', clientId: 'c', clientSecret: 's' },
|
||||
p2: { tenantId: 't', clientId: 'c', clientSecret: 's' },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const transformerA = jest.fn();
|
||||
const transformerB = jest.fn();
|
||||
|
||||
const providers = MicrosoftGraphIncrementalEntityProvider.fromConfig(
|
||||
config,
|
||||
{
|
||||
logger,
|
||||
userTransformer: { p1: transformerA, p2: transformerB },
|
||||
},
|
||||
);
|
||||
|
||||
expect(providers).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('around', () => {
|
||||
it('creates the client and passes it to the burst function', async () => {
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
let capturedContext: MSGraphContext | undefined;
|
||||
await provider.around(async ctx => {
|
||||
capturedContext = ctx;
|
||||
});
|
||||
|
||||
expect(MicrosoftGraphClient.create).toHaveBeenCalledWith(
|
||||
baseProviderConfig,
|
||||
);
|
||||
expect(capturedContext?.client).toBe(mockClient);
|
||||
});
|
||||
|
||||
it('applies providerConfigTransformer before creating the client', async () => {
|
||||
const transformedConfig = {
|
||||
...baseProviderConfig,
|
||||
clientSecret: 'rotated-secret',
|
||||
};
|
||||
const transformer = jest.fn().mockResolvedValue(transformedConfig);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
providerConfigTransformer: transformer,
|
||||
});
|
||||
|
||||
await provider.around(async () => {});
|
||||
|
||||
expect(transformer).toHaveBeenCalledWith(baseProviderConfig);
|
||||
expect(MicrosoftGraphClient.create).toHaveBeenCalledWith(
|
||||
transformedConfig,
|
||||
);
|
||||
});
|
||||
|
||||
it('warns when unsupported userGroupMember options are set', async () => {
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: {
|
||||
...baseProviderConfig,
|
||||
userGroupMemberFilter: 'some-filter',
|
||||
} as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
await provider.around(async () => {});
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'userGroupMemberFilter/Search/Path are not supported',
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('next — users phase', () => {
|
||||
it('starts in users phase when cursor is undefined', async () => {
|
||||
mockRequestOnePage.mockResolvedValue({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
expect(mockRequestOnePage).toHaveBeenCalledWith(
|
||||
mockClient,
|
||||
'users',
|
||||
expect.objectContaining({
|
||||
query: expect.objectContaining({ top: 999 }),
|
||||
}),
|
||||
);
|
||||
// No users → advances straight to groups phase
|
||||
expect(result.done).toBe(false);
|
||||
expect((result.cursor as MSGraphCursor).phase).toBe('groups');
|
||||
});
|
||||
|
||||
it('emits User entities with location annotations', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{
|
||||
id: 'user-id-1',
|
||||
displayName: 'Alice',
|
||||
mail: 'alice@example.com',
|
||||
userPrincipalName: 'alice@example.com',
|
||||
},
|
||||
],
|
||||
nextLink: 'https://graph.microsoft.com/v1.0/users?$skiptoken=page2',
|
||||
});
|
||||
mockGetUserPhotoGated.mockResolvedValue(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
expect(result.done).toBe(false);
|
||||
expect(result.entities).toHaveLength(1);
|
||||
|
||||
const entity = result.entities[0].entity;
|
||||
expect(entity.kind).toBe('User');
|
||||
expect(
|
||||
entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION],
|
||||
).toBe('user-id-1');
|
||||
expect(entity.metadata.annotations?.[ANNOTATION_LOCATION]).toBe(
|
||||
'msgraph:default/user-id-1',
|
||||
);
|
||||
expect(entity.metadata.annotations?.[ANNOTATION_ORIGIN_LOCATION]).toBe(
|
||||
'msgraph:default/user-id-1',
|
||||
);
|
||||
expect(result.entities[0].locationKey).toBe(
|
||||
'msgraph-org-provider:default',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns users cursor with nextLink when more pages remain', async () => {
|
||||
const nextLink =
|
||||
'https://graph.microsoft.com/v1.0/users?$skiptoken=page2';
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ id: 'u1', displayName: 'U1', userPrincipalName: 'u1@example.com' },
|
||||
],
|
||||
nextLink,
|
||||
});
|
||||
mockGetUserPhotoGated.mockResolvedValue(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
expect(result.done).toBe(false);
|
||||
expect((result.cursor as MSGraphCursor).phase).toBe('users');
|
||||
expect((result.cursor as MSGraphCursor).nextLink).toBe(nextLink);
|
||||
});
|
||||
|
||||
it('transitions to groups phase when last users page has no nextLink', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
expect(result.done).toBe(false);
|
||||
expect((result.cursor as MSGraphCursor).phase).toBe('groups');
|
||||
expect((result.cursor as MSGraphCursor).nextLink).toBeNull();
|
||||
});
|
||||
|
||||
it('skips users where the transformer returns undefined', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [{ id: 'u-no-name', userPrincipalName: '' }],
|
||||
nextLink: undefined,
|
||||
});
|
||||
mockGetUserPhotoGated.mockResolvedValue(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
userTransformer: async () => undefined,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
expect(result.entities).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('truncates entity names longer than 63 characters', async () => {
|
||||
const longUPN = 'a'.repeat(64) + '@example.com';
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [{ id: 'u1', displayName: 'Test', userPrincipalName: longUPN }],
|
||||
nextLink: undefined,
|
||||
});
|
||||
mockGetUserPhotoGated.mockResolvedValue(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
// Entity may be emitted or skipped depending on transformer handling of the UPN;
|
||||
// what matters is that no name exceeds 63 characters.
|
||||
for (const { entity } of result.entities) {
|
||||
expect(entity.metadata.name.length).toBeLessThanOrEqual(63);
|
||||
}
|
||||
});
|
||||
|
||||
it('skips photo loading when loadUserPhotos is false', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{
|
||||
id: 'u1',
|
||||
displayName: 'Alice',
|
||||
userPrincipalName: 'alice@example.com',
|
||||
},
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: { ...baseProviderConfig, loadUserPhotos: false } as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
await provider.next(
|
||||
makeContext({
|
||||
provider: { ...baseProviderConfig, loadUserPhotos: false } as any,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockGetUserPhotoGated).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('attempts photo loading when loadUserPhotos is not set', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{
|
||||
id: 'u1',
|
||||
displayName: 'Alice',
|
||||
userPrincipalName: 'alice@example.com',
|
||||
},
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
mockGetUserPhotoGated.mockResolvedValue(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
await provider.next(makeContext());
|
||||
|
||||
expect(mockGetUserPhotoGated).toHaveBeenCalledWith(mockClient, 'u1', 120);
|
||||
});
|
||||
|
||||
it('continues processing remaining users when a photo load fails', async () => {
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{
|
||||
id: 'u1',
|
||||
displayName: 'Alice',
|
||||
userPrincipalName: 'alice@example.com',
|
||||
},
|
||||
{
|
||||
id: 'u2',
|
||||
displayName: 'Bob',
|
||||
userPrincipalName: 'bob@example.com',
|
||||
},
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
mockGetUserPhotoGated
|
||||
.mockRejectedValueOnce(new Error('Photo service unavailable'))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext());
|
||||
|
||||
// Both users should still be emitted despite the photo failure
|
||||
expect(result.entities).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('next — groups phase', () => {
|
||||
const groupsCursor: MSGraphCursor = { phase: 'groups', nextLink: null };
|
||||
|
||||
it('emits the tenant root group on the first groups page', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
const rootGroup = result.entities.find(
|
||||
e =>
|
||||
e.entity.metadata.annotations?.[
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
|
||||
] === undefined && e.entity.spec?.type === 'root',
|
||||
);
|
||||
// Root group is emitted (org entity kind=Group, type=root)
|
||||
expect(result.entities.some(e => e.entity.spec?.type === 'root')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('does NOT emit the root group on subsequent pages', async () => {
|
||||
const continuationCursor: MSGraphCursor = {
|
||||
phase: 'groups',
|
||||
nextLink: 'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2',
|
||||
};
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
await provider.next(makeContext(), continuationCursor);
|
||||
|
||||
expect(mockClient.getOrganization).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('emits Group entities with location annotations', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' },
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
const groupEntity = result.entities.find(
|
||||
e =>
|
||||
e.entity.metadata.annotations?.[
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
|
||||
] === 'grp-1',
|
||||
);
|
||||
expect(groupEntity).toBeDefined();
|
||||
expect(
|
||||
groupEntity!.entity.metadata.annotations?.[ANNOTATION_LOCATION],
|
||||
).toBe('msgraph:default/grp-1');
|
||||
expect(
|
||||
groupEntity!.entity.metadata.annotations?.[ANNOTATION_ORIGIN_LOCATION],
|
||||
).toBe('msgraph:default/grp-1');
|
||||
expect(groupEntity!.locationKey).toBe('msgraph-org-provider:default');
|
||||
});
|
||||
|
||||
it('populates spec.members with user refs from getGroupMembers', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' },
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(
|
||||
asyncYield(
|
||||
{
|
||||
'@odata.type': '#microsoft.graph.user',
|
||||
id: 'u1',
|
||||
displayName: 'Alice',
|
||||
userPrincipalName: 'alice@example.com',
|
||||
},
|
||||
{
|
||||
'@odata.type': '#microsoft.graph.user',
|
||||
id: 'u2',
|
||||
displayName: 'Bob',
|
||||
userPrincipalName: 'bob@example.com',
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
const groupEntity = result.entities.find(
|
||||
e =>
|
||||
e.entity.metadata.annotations?.[
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
|
||||
] === 'grp-1',
|
||||
);
|
||||
expect(groupEntity!.entity.spec?.members).toHaveLength(2);
|
||||
expect(groupEntity!.entity.spec?.members).toContain(
|
||||
'user:default/alice_example.com',
|
||||
);
|
||||
});
|
||||
|
||||
it('populates spec.children with nested group refs from getGroupMembers', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{
|
||||
id: 'grp-parent',
|
||||
displayName: 'Parent',
|
||||
mail: 'parent@example.com',
|
||||
},
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(
|
||||
asyncYield({
|
||||
'@odata.type': '#microsoft.graph.group',
|
||||
id: 'grp-child',
|
||||
displayName: 'Child Group',
|
||||
mail: 'child@example.com',
|
||||
}),
|
||||
);
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
const parentEntity = result.entities.find(
|
||||
e =>
|
||||
e.entity.metadata.annotations?.[
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
|
||||
] === 'grp-parent',
|
||||
);
|
||||
expect(parentEntity!.entity.spec?.children).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns done:false with groups cursor when nextLink is present', async () => {
|
||||
const nextLink =
|
||||
'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2';
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({ items: [], nextLink });
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
expect(result.done).toBe(false);
|
||||
expect((result.cursor as MSGraphCursor).phase).toBe('groups');
|
||||
expect((result.cursor as MSGraphCursor).nextLink).toBe(nextLink);
|
||||
});
|
||||
|
||||
it('returns done:true when the last groups page has no nextLink', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
expect(result.done).toBe(true);
|
||||
});
|
||||
|
||||
it('continues processing remaining groups when root group fetch fails', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockRejectedValue(
|
||||
new Error('Organization not found'),
|
||||
);
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' },
|
||||
],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('failed to read organization root group'),
|
||||
expect.anything(),
|
||||
);
|
||||
// The group itself is still emitted
|
||||
expect(
|
||||
result.entities.some(
|
||||
e =>
|
||||
e.entity.metadata.annotations?.[
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
|
||||
] === 'grp-1',
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('skips groups where the transformer returns undefined', async () => {
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [{ id: 'grp-1', displayName: 'Engineering' }],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: baseProviderConfig as any,
|
||||
logger,
|
||||
groupTransformer: async () => undefined,
|
||||
});
|
||||
|
||||
const result = await provider.next(makeContext(), groupsCursor);
|
||||
|
||||
// Only the root group entity remains
|
||||
expect(result.entities.every(e => e.entity.spec?.type === 'root')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('passes group filter and search from provider config', async () => {
|
||||
const providerWithFilter = {
|
||||
...baseProviderConfig,
|
||||
groupFilter: 'securityEnabled eq true',
|
||||
groupSearch: '"displayName:Engineering"',
|
||||
};
|
||||
(mockClient.getOrganization as jest.Mock).mockResolvedValue({
|
||||
id: 'org-id',
|
||||
displayName: 'My Org',
|
||||
});
|
||||
mockRequestOnePage.mockResolvedValueOnce({
|
||||
items: [],
|
||||
nextLink: undefined,
|
||||
});
|
||||
(mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield());
|
||||
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: 'default',
|
||||
provider: providerWithFilter as any,
|
||||
logger,
|
||||
});
|
||||
|
||||
await provider.next(
|
||||
{ client: mockClient, provider: providerWithFilter as any },
|
||||
groupsCursor,
|
||||
);
|
||||
|
||||
expect(mockRequestOnePage).toHaveBeenCalledWith(
|
||||
mockClient,
|
||||
'groups',
|
||||
expect.objectContaining({
|
||||
query: expect.objectContaining({
|
||||
filter: 'securityEnabled eq true',
|
||||
search: '"displayName:Engineering"',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
+474
@@ -0,0 +1,474 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
Entity,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
IncrementalEntityProvider,
|
||||
EntityIteratorResult,
|
||||
} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { DeferredEntity } from '@backstage/plugin-catalog-node';
|
||||
import limiterFactory from 'p-limit';
|
||||
import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
|
||||
import {
|
||||
GroupTransformer,
|
||||
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
|
||||
MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
|
||||
MICROSOFT_GRAPH_USER_ID_ANNOTATION,
|
||||
MicrosoftGraphClient,
|
||||
MicrosoftGraphProviderConfig,
|
||||
OrganizationTransformer,
|
||||
ProviderConfigTransformer,
|
||||
UserTransformer,
|
||||
defaultGroupTransformer,
|
||||
defaultOrganizationTransformer,
|
||||
defaultUserTransformer,
|
||||
readProviderConfigs,
|
||||
} from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
import { getUserPhotoGated, requestOnePage } from './clientHelpers';
|
||||
|
||||
const PAGE_SIZE = 999;
|
||||
|
||||
/**
|
||||
* Backstage entity names must be ≤63 chars ([a-zA-Z0-9] separated by [-_.]).
|
||||
* When MS Graph UPNs exceed that (e.g. calendar/booking accounts), we truncate
|
||||
* to 54 chars and append an 8-char SHA-1 hash to preserve uniqueness.
|
||||
*/
|
||||
function capEntityName(name: string): string {
|
||||
if (name.length <= 63) return name;
|
||||
const hash = crypto.createHash('sha1').update(name).digest('hex').slice(0, 8);
|
||||
return `${name.slice(0, 54)}_${hash}`;
|
||||
}
|
||||
|
||||
/** Stamps `annotations.backstage.io/location` on an entity using the MS Graph UID. */
|
||||
function withLocations(providerId: string, entity: Entity): Entity {
|
||||
const uid =
|
||||
entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] ||
|
||||
entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ||
|
||||
entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] ||
|
||||
entity.metadata.name;
|
||||
const location = `msgraph:${providerId}/${encodeURIComponent(uid)}`;
|
||||
return {
|
||||
...entity,
|
||||
metadata: {
|
||||
...entity.metadata,
|
||||
annotations: {
|
||||
...entity.metadata.annotations,
|
||||
[ANNOTATION_LOCATION]: location,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: location,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pagination cursor used by {@link MicrosoftGraphIncrementalEntityProvider}.
|
||||
*
|
||||
* The `nextLink` field holds the `@odata.nextLink` URL returned by the
|
||||
* Microsoft Graph API, which encodes all state needed to resume a paged
|
||||
* request. A `null` value means the current phase is starting fresh.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type MSGraphCursor = {
|
||||
phase: 'users' | 'groups';
|
||||
nextLink: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Context passed to each burst of {@link MicrosoftGraphIncrementalEntityProvider}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type MSGraphContext = {
|
||||
client: MicrosoftGraphClient;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
};
|
||||
|
||||
/**
|
||||
* Options for {@link MicrosoftGraphIncrementalEntityProvider}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface MicrosoftGraphIncrementalEntityProviderOptions {
|
||||
/**
|
||||
* The logger to use.
|
||||
*/
|
||||
logger: LoggerService;
|
||||
|
||||
/**
|
||||
* The function that transforms a user entry in msgraph to an entity.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
userTransformer?: UserTransformer | Record<string, UserTransformer>;
|
||||
|
||||
/**
|
||||
* The function that transforms a group entry in msgraph to an entity.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
|
||||
|
||||
/**
|
||||
* The function that transforms an organization entry in msgraph to an entity.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
organizationTransformer?:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>;
|
||||
|
||||
/**
|
||||
* The function that transforms provider config dynamically before each sync.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
providerConfigTransformer?:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Incrementally reads user and group entries out of Microsoft Graph, one page
|
||||
* at a time, and provides them as User and Group entities for the catalog.
|
||||
*
|
||||
* Unlike `MicrosoftGraphOrgEntityProvider`, this provider never holds the full
|
||||
* dataset in memory at once. Each burst processes a single page of users or
|
||||
* groups (up to 999 items). This makes it suitable for very large tenants and
|
||||
* avoids the memory pressure and long-running task issues of the full-scan
|
||||
* provider.
|
||||
*
|
||||
* The Microsoft Graph `@odata.nextLink` URL is stored as the cursor, so a pod
|
||||
* restart during ingestion resumes from the last completed page.
|
||||
*
|
||||
* Group membership (`spec.members`) is resolved inline during the groups phase
|
||||
* by fetching the direct members of each group. The catalog's built-in relation
|
||||
* stitching derives `spec.memberOf` on users from these group membership lists.
|
||||
*
|
||||
* @remarks
|
||||
* `userGroupMemberFilter`, `userGroupMemberSearch`, and `userGroupMemberPath`
|
||||
* are not supported. Use `userFilter` / `userPath` to restrict which users are
|
||||
* ingested, and `groupFilter` / `groupSearch` to restrict which groups.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class MicrosoftGraphIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<MSGraphCursor, MSGraphContext>
|
||||
{
|
||||
/**
|
||||
* Create one provider instance per provider entry in
|
||||
* `catalog.providers.microsoftGraphOrg`.
|
||||
*/
|
||||
static fromConfig(
|
||||
configRoot: Config,
|
||||
options: MicrosoftGraphIncrementalEntityProviderOptions,
|
||||
): MicrosoftGraphIncrementalEntityProvider[] {
|
||||
function getTransformer<T extends Function>(
|
||||
id: string,
|
||||
transformers?: T | Record<string, T>,
|
||||
): T | undefined {
|
||||
if (['undefined', 'function'].includes(typeof transformers)) {
|
||||
return transformers as T;
|
||||
}
|
||||
return (transformers as Record<string, T>)[id];
|
||||
}
|
||||
|
||||
return readProviderConfigs(configRoot).map(
|
||||
providerConfig =>
|
||||
new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: providerConfig.id,
|
||||
provider: providerConfig,
|
||||
logger: options.logger,
|
||||
userTransformer: getTransformer(
|
||||
providerConfig.id,
|
||||
options.userTransformer,
|
||||
),
|
||||
groupTransformer: getTransformer(
|
||||
providerConfig.id,
|
||||
options.groupTransformer,
|
||||
),
|
||||
organizationTransformer: getTransformer(
|
||||
providerConfig.id,
|
||||
options.organizationTransformer,
|
||||
),
|
||||
providerConfigTransformer: getTransformer(
|
||||
providerConfig.id,
|
||||
options.providerConfigTransformer,
|
||||
),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private readonly options: {
|
||||
id: string;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
logger: LoggerService;
|
||||
userTransformer?: UserTransformer;
|
||||
groupTransformer?: GroupTransformer;
|
||||
organizationTransformer?: OrganizationTransformer;
|
||||
providerConfigTransformer?: ProviderConfigTransformer;
|
||||
};
|
||||
|
||||
constructor(options: {
|
||||
id: string;
|
||||
provider: MicrosoftGraphProviderConfig;
|
||||
logger: LoggerService;
|
||||
userTransformer?: UserTransformer;
|
||||
groupTransformer?: GroupTransformer;
|
||||
organizationTransformer?: OrganizationTransformer;
|
||||
providerConfigTransformer?: ProviderConfigTransformer;
|
||||
}) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.getProviderName} */
|
||||
getProviderName(): string {
|
||||
return `MicrosoftGraphIncrementalEntityProvider:${this.options.id}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the Microsoft Graph client for the duration of a full ingestion
|
||||
* cycle. The optional `providerConfigTransformer` is applied here so that
|
||||
* dynamic config changes (e.g., rotating credentials) take effect at the
|
||||
* start of each cycle rather than mid-way through.
|
||||
*/
|
||||
async around(
|
||||
burst: (context: MSGraphContext) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const provider = this.options.providerConfigTransformer
|
||||
? await this.options.providerConfigTransformer(this.options.provider)
|
||||
: this.options.provider;
|
||||
|
||||
if (
|
||||
provider.userGroupMemberFilter ||
|
||||
provider.userGroupMemberSearch ||
|
||||
provider.userGroupMemberPath
|
||||
) {
|
||||
this.options.logger.warn(
|
||||
`${this.getProviderName()}: userGroupMemberFilter/Search/Path are not supported by ` +
|
||||
`MicrosoftGraphIncrementalEntityProvider. Users will be fetched via the standard ` +
|
||||
`userFilter/userPath options instead. Switch to MicrosoftGraphOrgEntityProvider if ` +
|
||||
`you require userGroupMember-based ingestion.`,
|
||||
);
|
||||
}
|
||||
|
||||
const client = MicrosoftGraphClient.create(provider);
|
||||
await burst({ client, provider });
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.next} */
|
||||
async next(
|
||||
{ client, provider }: MSGraphContext,
|
||||
cursor?: MSGraphCursor,
|
||||
): Promise<EntityIteratorResult<MSGraphCursor>> {
|
||||
const phase = cursor?.phase ?? 'users';
|
||||
const nextLink = cursor?.nextLink ?? null;
|
||||
|
||||
if (phase === 'users') {
|
||||
return this.readUsersPage(client, provider, nextLink);
|
||||
}
|
||||
return this.readGroupsPage(client, provider, nextLink);
|
||||
}
|
||||
|
||||
private async readUsersPage(
|
||||
client: MicrosoftGraphClient,
|
||||
provider: MicrosoftGraphProviderConfig,
|
||||
nextLink: string | null,
|
||||
): Promise<EntityIteratorResult<MSGraphCursor>> {
|
||||
const { items: rawUsers, nextLink: newNextLink } =
|
||||
await requestOnePage<MicrosoftGraph.User>(
|
||||
client,
|
||||
provider.userPath ?? 'users',
|
||||
{
|
||||
query: {
|
||||
filter: provider.userFilter,
|
||||
expand: provider.userExpand,
|
||||
select: provider.userSelect,
|
||||
top: PAGE_SIZE,
|
||||
},
|
||||
queryMode: provider.queryMode,
|
||||
nextLink: nextLink ?? undefined,
|
||||
},
|
||||
);
|
||||
|
||||
const transformer = this.options.userTransformer ?? defaultUserTransformer;
|
||||
const limiter = limiterFactory(10);
|
||||
const entities: DeferredEntity[] = [];
|
||||
|
||||
await Promise.all(
|
||||
rawUsers.map(user =>
|
||||
limiter(async () => {
|
||||
let userPhoto: string | undefined;
|
||||
if (provider.loadUserPhotos !== false) {
|
||||
try {
|
||||
userPhoto = await getUserPhotoGated(client, user.id!, 120);
|
||||
} catch {
|
||||
// Photo load failures are non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
const entity = await transformer(user, userPhoto);
|
||||
if (entity) {
|
||||
entity.metadata.name = capEntityName(entity.metadata.name);
|
||||
entities.push({
|
||||
locationKey: `msgraph-org-provider:${this.options.id}`,
|
||||
entity: withLocations(this.options.id, entity),
|
||||
});
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
this.options.logger.debug(
|
||||
`${this.getProviderName()}: read ${entities.length} users`,
|
||||
{ phase: 'users', hasNextPage: !!newNextLink },
|
||||
);
|
||||
|
||||
if (newNextLink) {
|
||||
return {
|
||||
done: false,
|
||||
entities,
|
||||
cursor: { phase: 'users', nextLink: newNextLink },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
done: false,
|
||||
entities,
|
||||
cursor: { phase: 'groups', nextLink: null },
|
||||
};
|
||||
}
|
||||
|
||||
private async readGroupsPage(
|
||||
client: MicrosoftGraphClient,
|
||||
provider: MicrosoftGraphProviderConfig,
|
||||
nextLink: string | null,
|
||||
): Promise<EntityIteratorResult<MSGraphCursor>> {
|
||||
const { items: rawGroups, nextLink: newNextLink } =
|
||||
await requestOnePage<MicrosoftGraph.Group>(
|
||||
client,
|
||||
provider.groupPath ?? 'groups',
|
||||
{
|
||||
query: {
|
||||
filter: provider.groupFilter,
|
||||
search: provider.groupSearch,
|
||||
expand: provider.groupExpand,
|
||||
select: provider.groupSelect,
|
||||
top: PAGE_SIZE,
|
||||
},
|
||||
queryMode: provider.queryMode,
|
||||
nextLink: nextLink ?? undefined,
|
||||
},
|
||||
);
|
||||
|
||||
const groupTransformer =
|
||||
this.options.groupTransformer ?? defaultGroupTransformer;
|
||||
const userTransformer =
|
||||
this.options.userTransformer ?? defaultUserTransformer;
|
||||
const limiter = limiterFactory(10);
|
||||
const entities: DeferredEntity[] = [];
|
||||
|
||||
// Emit the tenant root group on the very first groups page
|
||||
if (!nextLink) {
|
||||
try {
|
||||
const organization = await client.getOrganization(provider.tenantId);
|
||||
const orgTransformer =
|
||||
this.options.organizationTransformer ??
|
||||
defaultOrganizationTransformer;
|
||||
const rootGroup = await orgTransformer(organization);
|
||||
if (rootGroup) {
|
||||
entities.push({
|
||||
locationKey: `msgraph-org-provider:${this.options.id}`,
|
||||
entity: withLocations(this.options.id, rootGroup),
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
this.options.logger.warn(
|
||||
`${this.getProviderName()}: failed to read organization root group`,
|
||||
{ error: e },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
rawGroups.map(group =>
|
||||
limiter(async () => {
|
||||
const entity = await groupTransformer(group);
|
||||
if (!entity) {
|
||||
return;
|
||||
}
|
||||
entity.metadata.name = capEntityName(entity.metadata.name);
|
||||
|
||||
const userRefs: string[] = [];
|
||||
const childRefs: string[] = [];
|
||||
|
||||
for await (const member of client.getGroupMembers(group.id!, {
|
||||
top: PAGE_SIZE,
|
||||
})) {
|
||||
if (member['@odata.type'] === '#microsoft.graph.user') {
|
||||
const userEntity = await userTransformer(
|
||||
member as MicrosoftGraph.User,
|
||||
);
|
||||
if (userEntity) {
|
||||
userEntity.metadata.name = capEntityName(
|
||||
userEntity.metadata.name,
|
||||
);
|
||||
userRefs.push(stringifyEntityRef(userEntity));
|
||||
}
|
||||
} else if (member['@odata.type'] === '#microsoft.graph.group') {
|
||||
const childEntity = await groupTransformer(
|
||||
member as MicrosoftGraph.Group,
|
||||
);
|
||||
if (childEntity) {
|
||||
childEntity.metadata.name = capEntityName(
|
||||
childEntity.metadata.name,
|
||||
);
|
||||
childRefs.push(stringifyEntityRef(childEntity));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entity.spec.members = userRefs;
|
||||
entity.spec.children = childRefs;
|
||||
|
||||
entities.push({
|
||||
locationKey: `msgraph-org-provider:${this.options.id}`,
|
||||
entity: withLocations(this.options.id, entity),
|
||||
});
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
this.options.logger.debug(
|
||||
`${this.getProviderName()}: read ${rawGroups.length} groups`,
|
||||
{ phase: 'groups', hasNextPage: !!newNextLink },
|
||||
);
|
||||
|
||||
if (newNextLink) {
|
||||
return {
|
||||
done: false,
|
||||
entities,
|
||||
cursor: { phase: 'groups', nextLink: newNextLink },
|
||||
};
|
||||
}
|
||||
|
||||
return { done: true, entities };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Re-export everything from the main entry point while the package is in alpha.
|
||||
export * from './index';
|
||||
export { default } from './index';
|
||||
@@ -0,0 +1,254 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { getUserPhotoGated, requestOnePage } from './clientHelpers';
|
||||
|
||||
function makeResponse(status: number, body: unknown): Response {
|
||||
return {
|
||||
status,
|
||||
json: () => Promise.resolve(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe('requestOnePage', () => {
|
||||
const client = {
|
||||
requestApi: jest.fn(),
|
||||
requestRaw: jest.fn(),
|
||||
} as unknown as jest.Mocked<MicrosoftGraphClient>;
|
||||
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('fetches a basic page using requestApi', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, {
|
||||
value: [{ id: '1', displayName: 'Alice' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await requestOnePage<{ id: string }>(client, 'users', {
|
||||
query: { top: 999 },
|
||||
});
|
||||
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.items[0].id).toBe('1');
|
||||
expect(result.nextLink).toBeUndefined();
|
||||
expect(client.requestApi).toHaveBeenCalledWith(
|
||||
'users',
|
||||
{ top: 999 },
|
||||
{},
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('adds ConsistencyLevel and $count for advanced mode with filter', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, { value: [] }),
|
||||
);
|
||||
|
||||
await requestOnePage(client, 'users', {
|
||||
query: {
|
||||
filter: 'accountEnabled eq true and jobTitle ne null',
|
||||
top: 999,
|
||||
},
|
||||
queryMode: 'advanced',
|
||||
});
|
||||
|
||||
expect(client.requestApi).toHaveBeenCalledWith(
|
||||
'users',
|
||||
expect.objectContaining({ count: true }),
|
||||
{ ConsistencyLevel: 'eventual' },
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('auto-promotes to advanced mode when $search is present', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, { value: [] }),
|
||||
);
|
||||
|
||||
await requestOnePage(client, 'groups', {
|
||||
query: { search: '"displayName:Sales"', top: 10 },
|
||||
});
|
||||
|
||||
expect(client.requestApi).toHaveBeenCalledWith(
|
||||
'groups',
|
||||
expect.objectContaining({ count: true }),
|
||||
{ ConsistencyLevel: 'eventual' },
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not set $count for basic mode', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, { value: [] }),
|
||||
);
|
||||
|
||||
await requestOnePage(client, 'users', {
|
||||
query: { filter: 'accountEnabled eq true', top: 999 },
|
||||
queryMode: 'basic',
|
||||
});
|
||||
|
||||
expect(client.requestApi).toHaveBeenCalledWith(
|
||||
'users',
|
||||
expect.not.objectContaining({ count: true }),
|
||||
{},
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('follows nextLink using requestRaw', async () => {
|
||||
const nextLink =
|
||||
'https://graph.microsoft.com/v1.0/users?$skiptoken=abc123&$count=true';
|
||||
(client.requestRaw as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, {
|
||||
value: [{ id: '2' }],
|
||||
'@odata.nextLink':
|
||||
'https://graph.microsoft.com/v1.0/users?$skiptoken=def456',
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await requestOnePage(client, 'users', { nextLink });
|
||||
|
||||
expect(client.requestRaw).toHaveBeenCalledWith(nextLink, {}, 2, undefined);
|
||||
expect(client.requestApi).not.toHaveBeenCalled();
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.nextLink).toBe(
|
||||
'https://graph.microsoft.com/v1.0/users?$skiptoken=def456',
|
||||
);
|
||||
});
|
||||
|
||||
it('passes ConsistencyLevel header when following nextLink in advanced mode', async () => {
|
||||
const nextLink = 'https://graph.microsoft.com/v1.0/groups?$skiptoken=xyz';
|
||||
(client.requestRaw as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, { value: [] }),
|
||||
);
|
||||
|
||||
await requestOnePage(client, 'groups', {
|
||||
query: { filter: 'securityEnabled eq true', top: 999 },
|
||||
queryMode: 'advanced',
|
||||
nextLink,
|
||||
});
|
||||
|
||||
expect(client.requestRaw).toHaveBeenCalledWith(
|
||||
nextLink,
|
||||
{ ConsistencyLevel: 'eventual' },
|
||||
2,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('returns nextLink from response when present', async () => {
|
||||
const expectedNextLink =
|
||||
'https://graph.microsoft.com/v1.0/users?$skiptoken=page2';
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, {
|
||||
value: Array(999).fill({ id: 'x' }),
|
||||
'@odata.nextLink': expectedNextLink,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await requestOnePage(client, 'users', {});
|
||||
|
||||
expect(result.nextLink).toBe(expectedNextLink);
|
||||
expect(result.items).toHaveLength(999);
|
||||
});
|
||||
|
||||
it('throws a descriptive error on non-200 response', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(403, {
|
||||
error: {
|
||||
code: 'Authorization_RequestDenied',
|
||||
message: 'Access denied',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(requestOnePage(client, 'users', {})).rejects.toThrow(
|
||||
'Authorization_RequestDenied - Access denied',
|
||||
);
|
||||
});
|
||||
|
||||
it('passes AbortSignal through to the client', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue(
|
||||
makeResponse(200, { value: [] }),
|
||||
);
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
await requestOnePage(client, 'users', { signal });
|
||||
|
||||
expect(client.requestApi).toHaveBeenCalledWith(
|
||||
'users',
|
||||
undefined,
|
||||
{},
|
||||
signal,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getUserPhotoGated', () => {
|
||||
const client = {
|
||||
requestApi: jest.fn(),
|
||||
getUserPhotoWithSizeLimit: jest.fn(),
|
||||
} as unknown as jest.Mocked<MicrosoftGraphClient>;
|
||||
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('returns undefined when the photo check returns non-200', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue({
|
||||
status: 404,
|
||||
} as Response);
|
||||
|
||||
const result = await getUserPhotoGated(client, 'user-id', 120);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns the photo data URI when the check passes', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue({
|
||||
status: 200,
|
||||
} as Response);
|
||||
(client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue(
|
||||
'data:image/jpeg;base64,/9j/abc123',
|
||||
);
|
||||
|
||||
const result = await getUserPhotoGated(client, 'user-id', 120);
|
||||
|
||||
expect(result).toBe('data:image/jpeg;base64,/9j/abc123');
|
||||
expect(client.requestApi).toHaveBeenCalledWith('users/user-id/photo');
|
||||
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
|
||||
'user-id',
|
||||
120,
|
||||
);
|
||||
});
|
||||
|
||||
it('passes the maxSize limit to getUserPhotoWithSizeLimit', async () => {
|
||||
(client.requestApi as jest.Mock).mockResolvedValue({
|
||||
status: 200,
|
||||
} as Response);
|
||||
(client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue(
|
||||
undefined,
|
||||
);
|
||||
|
||||
await getUserPhotoGated(client, 'user-abc', 48);
|
||||
|
||||
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
|
||||
'user-abc',
|
||||
48,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
MicrosoftGraphClient,
|
||||
ODataQuery,
|
||||
} from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
|
||||
/**
|
||||
* Fetches a single page of Graph API results.
|
||||
*
|
||||
* When `options.nextLink` is provided it is followed directly (all query
|
||||
* parameters are already encoded in it). Otherwise the request is built from
|
||||
* `options.query`.
|
||||
*
|
||||
* MS Graph requires `ConsistencyLevel: eventual` + `$count=true` for advanced
|
||||
* queries using `ne`/`not` operators in `$filter` or using `$search`.
|
||||
*/
|
||||
export async function requestOnePage<T>(
|
||||
client: MicrosoftGraphClient,
|
||||
path: string,
|
||||
options: {
|
||||
query?: ODataQuery;
|
||||
queryMode?: 'basic' | 'advanced';
|
||||
nextLink?: string;
|
||||
signal?: AbortSignal;
|
||||
} = {},
|
||||
): Promise<{ items: T[]; nextLink?: string }> {
|
||||
const { query, queryMode, nextLink, signal } = options;
|
||||
const appliedQueryMode = query?.search ? 'advanced' : queryMode ?? 'basic';
|
||||
|
||||
// $count=true is required for advanced queries using ne/not in $filter or $search
|
||||
if (
|
||||
appliedQueryMode === 'advanced' &&
|
||||
query &&
|
||||
(query.filter || query.search)
|
||||
) {
|
||||
query.count = true;
|
||||
}
|
||||
|
||||
const headers: Record<string, string> =
|
||||
appliedQueryMode === 'advanced' ? { ConsistencyLevel: 'eventual' } : {};
|
||||
|
||||
const response = nextLink
|
||||
? await client.requestRaw(nextLink, headers, 2, signal)
|
||||
: await client.requestApi(path, query, headers, signal);
|
||||
|
||||
if (response.status !== 200) {
|
||||
const body = await response.json();
|
||||
const err = body?.error;
|
||||
throw new Error(
|
||||
`Error while reading ${nextLink ?? path} from Microsoft Graph: ${
|
||||
err?.code
|
||||
} - ${err?.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return {
|
||||
items: result.value as T[],
|
||||
nextLink: result['@odata.nextLink'],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `getUserPhotoWithSizeLimit` but skips the size-listing call for users
|
||||
* with no photo. For users without a photo: 1 fast check call. For users with
|
||||
* a photo: 1 check + the normal size-limited fetch (2 more calls).
|
||||
*/
|
||||
export async function getUserPhotoGated(
|
||||
client: MicrosoftGraphClient,
|
||||
userId: string,
|
||||
maxSize: number,
|
||||
): Promise<string | undefined> {
|
||||
const check = await client.requestApi(`users/${userId}/photo`);
|
||||
if (check.status !== 200) return undefined;
|
||||
return await client.getUserPhotoWithSizeLimit(userId, maxSize);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A Backstage catalog backend module that incrementally ingests users and
|
||||
* groups from Microsoft Graph, one page at a time.
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
export { catalogModuleMicrosoftGraphIncrementalEntityProvider as default } from './module';
|
||||
export * from './module';
|
||||
export { MicrosoftGraphIncrementalEntityProvider } from './MicrosoftGraphIncrementalEntityProvider';
|
||||
export type {
|
||||
MSGraphCursor,
|
||||
MSGraphContext,
|
||||
MicrosoftGraphIncrementalEntityProviderOptions,
|
||||
} from './MicrosoftGraphIncrementalEntityProvider';
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
|
||||
import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import { catalogModuleMicrosoftGraphIncrementalEntityProvider } from './catalogModuleMicrosoftGraphIncrementalEntityProvider';
|
||||
import { MicrosoftGraphIncrementalEntityProvider } from '../MicrosoftGraphIncrementalEntityProvider';
|
||||
|
||||
describe('catalogModuleMicrosoftGraphIncrementalEntityProvider', () => {
|
||||
it('registers the provider at the incremental ingestion extension point', async () => {
|
||||
const addProvider = jest.fn();
|
||||
const extensionPoint = { addProvider };
|
||||
|
||||
await startTestBackend({
|
||||
extensionPoints: [
|
||||
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
|
||||
],
|
||||
features: [
|
||||
catalogModuleMicrosoftGraphIncrementalEntityProvider,
|
||||
mockServices.rootConfig.factory({
|
||||
data: {
|
||||
catalog: {
|
||||
providers: {
|
||||
microsoftGraphOrg: {
|
||||
default: {
|
||||
tenantId: 'tenant-id',
|
||||
clientId: 'client-id',
|
||||
clientSecret: 'client-secret',
|
||||
schedule: {
|
||||
frequency: 'PT12H',
|
||||
timeout: 'PT4H',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(addProvider).toHaveBeenCalledTimes(1);
|
||||
const { provider, options } = addProvider.mock.calls[0][0];
|
||||
expect(provider).toBeInstanceOf(MicrosoftGraphIncrementalEntityProvider);
|
||||
expect(provider.getProviderName()).toBe(
|
||||
'MicrosoftGraphIncrementalEntityProvider:default',
|
||||
);
|
||||
expect(options.burstInterval).toEqual({ seconds: 3 });
|
||||
expect(options.burstLength).toEqual({ minutes: 5 });
|
||||
// restLength derived from schedule.frequency (12h)
|
||||
expect(options.restLength).toEqual({ hours: 12 });
|
||||
});
|
||||
|
||||
it('creates one provider per config entry', async () => {
|
||||
const addProvider = jest.fn();
|
||||
const extensionPoint = { addProvider };
|
||||
|
||||
await startTestBackend({
|
||||
extensionPoints: [
|
||||
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
|
||||
],
|
||||
features: [
|
||||
catalogModuleMicrosoftGraphIncrementalEntityProvider,
|
||||
mockServices.rootConfig.factory({
|
||||
data: {
|
||||
catalog: {
|
||||
providers: {
|
||||
microsoftGraphOrg: {
|
||||
tenantA: {
|
||||
tenantId: 'a',
|
||||
clientId: 'c',
|
||||
clientSecret: 's',
|
||||
},
|
||||
tenantB: {
|
||||
tenantId: 'b',
|
||||
clientId: 'c',
|
||||
clientSecret: 's',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(addProvider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('defaults restLength to 8 hours when no schedule frequency is configured', async () => {
|
||||
const addProvider = jest.fn();
|
||||
const extensionPoint = { addProvider };
|
||||
|
||||
await startTestBackend({
|
||||
extensionPoints: [
|
||||
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
|
||||
],
|
||||
features: [
|
||||
catalogModuleMicrosoftGraphIncrementalEntityProvider,
|
||||
mockServices.rootConfig.factory({
|
||||
data: {
|
||||
catalog: {
|
||||
providers: {
|
||||
microsoftGraphOrg: {
|
||||
default: {
|
||||
tenantId: 'tenant-id',
|
||||
clientId: 'client-id',
|
||||
clientSecret: 'client-secret',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const { options } = addProvider.mock.calls[0][0];
|
||||
expect(options.restLength).toEqual({ hours: 8 });
|
||||
});
|
||||
});
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
coreServices,
|
||||
createBackendModule,
|
||||
createExtensionPoint,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
import {
|
||||
GroupTransformer,
|
||||
OrganizationTransformer,
|
||||
ProviderConfigTransformer,
|
||||
UserTransformer,
|
||||
readProviderConfigs,
|
||||
} from '@backstage/plugin-catalog-backend-module-msgraph';
|
||||
import { HumanDuration } from '@backstage/types';
|
||||
import {
|
||||
MicrosoftGraphIncrementalEntityProvider,
|
||||
MSGraphCursor,
|
||||
} from '../MicrosoftGraphIncrementalEntityProvider';
|
||||
|
||||
/**
|
||||
* Interface for
|
||||
* {@link microsoftGraphIncrementalEntityProviderTransformExtensionPoint}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint {
|
||||
/**
|
||||
* Set the function that transforms a user entry in msgraph to an entity.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
setUserTransformer(
|
||||
transformer: UserTransformer | Record<string, UserTransformer>,
|
||||
): void;
|
||||
|
||||
/**
|
||||
* Set the function that transforms a group entry in msgraph to an entity.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
setGroupTransformer(
|
||||
transformer: GroupTransformer | Record<string, GroupTransformer>,
|
||||
): void;
|
||||
|
||||
/**
|
||||
* Set the function that transforms an organization entry in msgraph to an
|
||||
* entity. Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
setOrganizationTransformer(
|
||||
transformer:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>,
|
||||
): void;
|
||||
|
||||
/**
|
||||
* Set the function that transforms provider config dynamically.
|
||||
* Optionally, you can pass separate transformers per provider ID.
|
||||
*/
|
||||
setProviderConfigTransformer(
|
||||
transformer:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>,
|
||||
): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extension point used to customize the transforms applied by the incremental
|
||||
* module.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint =
|
||||
createExtensionPoint<MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint>(
|
||||
{
|
||||
id: 'catalog.microsoftGraphIncrementalEntityProvider.transforms',
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Registers {@link MicrosoftGraphIncrementalEntityProvider} instances with the
|
||||
* catalog's incremental ingestion extension point.
|
||||
*
|
||||
* This module requires `catalogModuleIncrementalIngestionEntityProvider` to
|
||||
* also be installed in the backend.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* // packages/backend/src/index.ts
|
||||
* backend.add(import('@backstage/plugin-catalog-backend-module-incremental-ingestion'));
|
||||
* backend.add(import('@backstage/plugin-catalog-backend-module-msgraph-incremental'));
|
||||
* ```
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const catalogModuleMicrosoftGraphIncrementalEntityProvider =
|
||||
createBackendModule({
|
||||
pluginId: 'catalog',
|
||||
moduleId: 'microsoftGraphIncrementalEntityProvider',
|
||||
register(env) {
|
||||
let userTransformer:
|
||||
| UserTransformer
|
||||
| Record<string, UserTransformer>
|
||||
| undefined;
|
||||
let groupTransformer:
|
||||
| GroupTransformer
|
||||
| Record<string, GroupTransformer>
|
||||
| undefined;
|
||||
let organizationTransformer:
|
||||
| OrganizationTransformer
|
||||
| Record<string, OrganizationTransformer>
|
||||
| undefined;
|
||||
let providerConfigTransformer:
|
||||
| ProviderConfigTransformer
|
||||
| Record<string, ProviderConfigTransformer>
|
||||
| undefined;
|
||||
|
||||
env.registerExtensionPoint(
|
||||
microsoftGraphIncrementalEntityProviderTransformExtensionPoint,
|
||||
{
|
||||
setUserTransformer(transformer) {
|
||||
if (userTransformer) {
|
||||
throw new Error('User transformer may only be set once');
|
||||
}
|
||||
userTransformer = transformer;
|
||||
},
|
||||
setGroupTransformer(transformer) {
|
||||
if (groupTransformer) {
|
||||
throw new Error('Group transformer may only be set once');
|
||||
}
|
||||
groupTransformer = transformer;
|
||||
},
|
||||
setOrganizationTransformer(transformer) {
|
||||
if (organizationTransformer) {
|
||||
throw new Error('Organization transformer may only be set once');
|
||||
}
|
||||
organizationTransformer = transformer;
|
||||
},
|
||||
setProviderConfigTransformer(transformer) {
|
||||
if (providerConfigTransformer) {
|
||||
throw new Error(
|
||||
'Provider config transformer may only be set once',
|
||||
);
|
||||
}
|
||||
providerConfigTransformer = transformer;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
env.registerInit({
|
||||
deps: {
|
||||
config: coreServices.rootConfig,
|
||||
logger: coreServices.logger,
|
||||
incremental: incrementalIngestionProvidersExtensionPoint,
|
||||
},
|
||||
async init({ config, logger, incremental }) {
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
for (const providerConfig of providerConfigs) {
|
||||
const provider = new MicrosoftGraphIncrementalEntityProvider({
|
||||
id: providerConfig.id,
|
||||
provider: providerConfig,
|
||||
logger,
|
||||
userTransformer: resolveTransformer(
|
||||
providerConfig.id,
|
||||
userTransformer,
|
||||
),
|
||||
groupTransformer: resolveTransformer(
|
||||
providerConfig.id,
|
||||
groupTransformer,
|
||||
),
|
||||
organizationTransformer: resolveTransformer(
|
||||
providerConfig.id,
|
||||
organizationTransformer,
|
||||
),
|
||||
providerConfigTransformer: resolveTransformer(
|
||||
providerConfig.id,
|
||||
providerConfigTransformer,
|
||||
),
|
||||
});
|
||||
|
||||
const restLength = deriveRestLength(providerConfig);
|
||||
|
||||
incremental.addProvider<MSGraphCursor, unknown>({
|
||||
provider,
|
||||
options: {
|
||||
burstInterval: { seconds: 3 },
|
||||
burstLength: { minutes: 5 },
|
||||
restLength,
|
||||
backoff: [
|
||||
{ seconds: 30 },
|
||||
{ minutes: 3 },
|
||||
{ minutes: 30 },
|
||||
{ hours: 3 },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
function resolveTransformer<T extends Function>(
|
||||
id: string,
|
||||
transformer?: T | Record<string, T>,
|
||||
): T | undefined {
|
||||
if (['undefined', 'function'].includes(typeof transformer)) {
|
||||
return transformer as T;
|
||||
}
|
||||
return (transformer as Record<string, T>)[id];
|
||||
}
|
||||
|
||||
function deriveRestLength(
|
||||
providerConfig: ReturnType<typeof readProviderConfigs>[number],
|
||||
): HumanDuration {
|
||||
const freq = providerConfig.schedule?.frequency;
|
||||
if (freq && typeof freq === 'object' && !('cron' in freq)) {
|
||||
return freq as HumanDuration;
|
||||
}
|
||||
return { hours: 8 };
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright 2024 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export {
|
||||
catalogModuleMicrosoftGraphIncrementalEntityProvider,
|
||||
microsoftGraphIncrementalEntityProviderTransformExtensionPoint,
|
||||
type MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint,
|
||||
} from './catalogModuleMicrosoftGraphIncrementalEntityProvider';
|
||||
@@ -5016,7 +5016,7 @@ __metadata:
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion":
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:^, @backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion"
|
||||
dependencies:
|
||||
@@ -5072,7 +5072,24 @@ __metadata:
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
"@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph":
|
||||
"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental"
|
||||
dependencies:
|
||||
"@backstage/backend-plugin-api": "workspace:^"
|
||||
"@backstage/catalog-model": "workspace:^"
|
||||
"@backstage/cli": "workspace:^"
|
||||
"@backstage/config": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^"
|
||||
"@backstage/plugin-catalog-node": "workspace:^"
|
||||
"@backstage/types": "workspace:^"
|
||||
"@microsoft/microsoft-graph-types": "npm:^2.6.0"
|
||||
p-limit: "npm:^3.0.2"
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
"@backstage/plugin-catalog-backend-module-msgraph@workspace:^, @backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph"
|
||||
dependencies:
|
||||
@@ -29085,7 +29102,9 @@ __metadata:
|
||||
"@backstage/plugin-auth-node": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-logs": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-openapi": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^"
|
||||
@@ -29115,6 +29134,7 @@ __metadata:
|
||||
"@opentelemetry/exporter-prometheus": "npm:^0.213.0"
|
||||
"@opentelemetry/sdk-node": "npm:^0.213.0"
|
||||
example-app: "link:../app"
|
||||
pg: "npm:^8.11.0"
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
@@ -39927,7 +39947,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0":
|
||||
"pg@npm:^8.11.0, pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0":
|
||||
version: 8.20.0
|
||||
resolution: "pg@npm:8.20.0"
|
||||
dependencies:
|
||||
|
||||
Reference in New Issue
Block a user