diff --git a/.changeset/chilly-shoes-doubt.md b/.changeset/chilly-shoes-doubt.md new file mode 100644 index 0000000000..978600f5b4 --- /dev/null +++ b/.changeset/chilly-shoes-doubt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Fixed disabling of user photo fetching. Previously, the config value wasn't propagated properly, so user photos was still being fetched despite disabled by config. diff --git a/.changeset/silent-wombats-hang.md b/.changeset/silent-wombats-hang.md new file mode 100644 index 0000000000..85bda32007 --- /dev/null +++ b/.changeset/silent-wombats-hang.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Handle fetching huge amounts of users from Azure without crashing diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 38c2e7618a..5e39aaeb7d 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -146,6 +146,18 @@ microsoftGraphOrg: search: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' ``` +### User photos + +By default, the photos of users will be fetched and added to each user entity. For huge organizations this may be unfeasible, as it will take a _very_ long time, and can be disabled by setting `loadPhotos` to `false`: + +```yaml +microsoftGraphOrg: + providerId: + user: + filter: ... + loadPhotos: false +``` + ## Customizing Transformation Ingested entities can be customized by providing custom transformers. diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index e3ed302fa9..7ee416ecd4 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -54,6 +54,8 @@ catalog: # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter # This and userGroupMemberFilter are mutually exclusive, only one can be specified filter: accountEnabled eq true and userType eq 'member' + # Set to false to not load user photos. + loadPhotos: true # See https://docs.microsoft.com/en-us/graph/api/resources/schemaextension?view=graph-rest-1.0 select: ['id', 'displayName', 'description'] # Optional configuration block diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index b1dfc5a148..9f4b9ee08c 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -955,6 +955,43 @@ describe('read microsoft graph', () => { ); }); + it('should ignore loading photos if loadPhotos is false', async () => { + client.getOrganization.mockResolvedValue(getExampleOrg()); + + client.getUsers.mockImplementation(getExampleUsers); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + await readMicrosoftGraphOrg(client, 'tenantid', { + logger: getVoidLogger(), + loadUserPhotos: false, + }); + + expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(0); + + expect(client.getUsers).toHaveBeenCalledTimes(1); + expect(client.getUsers).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + expect(client.getGroups).toHaveBeenCalledTimes(1); + expect(client.getGroups).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + }); + it('should read users with userSelect', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', @@ -1027,5 +1064,51 @@ describe('read microsoft graph', () => { ); expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledTimes(1); }); + + it('should handle loading huge amounts of users', async () => { + client.getOrganization.mockResolvedValue(getExampleOrg()); + + const userCount = 200_000; + + async function* getHugeAmountsOfExampleUsers() { + for (let i = 0; i < userCount; ++i) { + yield { + id: `userid-${i}`, + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + } + + client.getUsers.mockImplementation(getHugeAmountsOfExampleUsers); + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphOrg(client, 'tenantid', { + logger: getVoidLogger(), + loadUserPhotos: false, + }); + + expect(users.length).toBe(userCount); + + expect(client.getUsers).toHaveBeenCalledTimes(1); + expect(client.getUsers).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + expect(client.getGroups).toHaveBeenCalledTimes(1); + expect(client.getGroups).toHaveBeenCalledWith( + { + top: 999, + }, + undefined, + ); + }); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 0f3d81c224..2ca21fca9c 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -384,7 +384,7 @@ export async function readMicrosoftGraphOrg( logger: LoggerService; }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { - const users: UserEntity[] = []; + let users: UserEntity[] = []; if (options.userGroupMemberFilter || options.userGroupMemberSearch) { const { users: usersInGroups } = await readMicrosoftGraphUsersInGroups( @@ -401,7 +401,7 @@ export async function readMicrosoftGraphOrg( logger: options.logger, }, ); - users.push(...usersInGroups); + users = usersInGroups; } else { const { users: usersWithFilter } = await readMicrosoftGraphUsers(client, { queryMode: options.queryMode, @@ -412,7 +412,7 @@ export async function readMicrosoftGraphOrg( transformer: options.userTransformer, logger: options.logger, }); - users.push(...usersWithFilter); + users = usersWithFilter; } const { groups, rootGroup, groupMember, groupMemberOf } = await readMicrosoftGraphGroups(client, tenantId, { diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 9c804216e4..3c8b903e69 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -310,6 +310,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { userExpand: provider.userExpand, userFilter: provider.userFilter, userSelect: provider.userSelect, + loadUserPhotos: provider.loadUserPhotos, userGroupMemberFilter: provider.userGroupMemberFilter, userGroupMemberSearch: provider.userGroupMemberSearch, groupExpand: provider.groupExpand, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 570959a381..c058286cb2 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -112,6 +112,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { { userExpand: provider.userExpand, userFilter: provider.userFilter, + loadUserPhotos: provider.loadUserPhotos, userGroupMemberFilter: provider.userGroupMemberFilter, userGroupMemberSearch: provider.userGroupMemberSearch, groupExpand: provider.groupExpand,