feat: add events support for gitlab catalog entity provider

Signed-off-by: ElaineDeMattosSilvaB <elaine.de-mattos-silva-bezerra@deutschebahn.com>

.changeset/eleven-pandas-divide.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab': patch
+---
+
+Added events support for `GitlabDiscoveryEntityProvider` and `GitlabOrgDiscoveryEntityProvider`.

.changeset/shy-students-clap.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-gitlab-org': patch
+---
+
+Added a new `catalog-backend-module-gitlab-org` module which adds the `GitlabOrgDiscoveryEntityProvider` to the catalog's providers using the new backend system.

plugins/catalog-backend-module-gitlab-org/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/catalog-backend-module-gitlab-org/README.md

@@ -0,0 +1,8 @@
+# Catalog Backend Module for GitLab Organizational Data
+
+This is an extension module to the plugin-catalog-backend plugin that provides group and user entities by scrapping or receiving events from a GitLab instance.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/gitlab/org) for details on how to install
+and configure the plugin.

plugins/catalog-backend-module-gitlab-org/api-report.md

@@ -0,0 +1,11 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-gitlab-org"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { BackendFeature } from '@backstage/backend-plugin-api';
+
+// @public
+const catalogModuleGitlabOrgDiscoveryEntityProvider: () => BackendFeature;
+export default catalogModuleGitlabOrgDiscoveryEntityProvider;
+```

plugins/catalog-backend-module-gitlab-org/catalog-info.yaml

@@ -0,0 +1,10 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: backstage-plugin-catalog-backend-module-gitlab-org
+  title: '@backstage/plugin-catalog-backend-module-gitlab-org'
+  description: The gitlab-org backend module for the catalog plugin.
+spec:
+  lifecycle: experimental
+  type: backstage-backend-plugin-module
+  owner: maintainers

plugins/catalog-backend-module-gitlab-org/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-gitlab-org",
+  "version": "0.0.0",
+  "description": "The gitlab-org backend module for the catalog plugin.",
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-gitlab-org"
+  },
+  "license": "Apache-2.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "clean": "backstage-cli package clean",
+    "lint": "backstage-cli package lint",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "start": "backstage-cli package start",
+    "test": "backstage-cli package test"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "workspace:^",
+    "@backstage/backend-plugin-api": "workspace:^",
+    "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^",
+    "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^"
+  },
+  "devDependencies": {
+    "@backstage/backend-tasks": "workspace:^",
+    "@backstage/backend-test-utils": "workspace:^",
+    "@backstage/cli": "workspace:^",
+    "@backstage/plugin-events-backend-test-utils": "workspace:^",
+    "luxon": "^3.0.0"
+  }
+}

plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createServiceFactory } from '@backstage/backend-plugin-api';
+import { TaskScheduleDefinition } from '@backstage/backend-tasks';
+import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { TestEventsService } from '@backstage/plugin-events-backend-test-utils';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+import { Duration } from 'luxon';
+import { catalogModuleGitlabOrgDiscoveryEntityProvider } from './catalogModuleGitlabOrgDiscoveryEntityProvider';
+
+describe('catalogModuleGitlabOrgDiscoveryEntityProvider', () => {
+  it('should register provider at the catalog extension point', async () => {
+    const events = new TestEventsService();
+    const eventsServiceFactory = createServiceFactory({
+      service: eventsServiceRef,
+      deps: {},
+      async factory({}) {
+        return events;
+      },
+    });
+    let addedProviders: Array<GitlabOrgDiscoveryEntityProvider> | undefined;
+    let usedSchedule: TaskScheduleDefinition | undefined;
+
+    const extensionPoint = {
+      addEntityProvider: (providers: any) => {
+        addedProviders = providers;
+      },
+    };
+    const connection = jest.fn() as unknown as EntityProviderConnection;
+    const runner = jest.fn();
+    const scheduler = mockServices.scheduler.mock({
+      createScheduledTaskRunner(schedule) {
+        usedSchedule = schedule;
+        return { run: runner };
+      },
+    });
+
+    const config = {
+      integrations: {
+        gitlab: [
+          {
+            host: 'test-gitlab',
+            apiBaseUrl: 'https://api.gitlab.example/api/v4',
+            token: '1234',
+          },
+        ],
+      },
+      catalog: {
+        providers: {
+          gitlab: {
+            'test-id': {
+              host: 'test-gitlab',
+              group: 'test-group',
+              orgEnabled: true,
+              schedule: {
+                frequency: 'P1M',
+                timeout: 'PT3M',
+              },
+            },
+          },
+        },
+      },
+    };
+
+    await startTestBackend({
+      extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
+      features: [
+        eventsServiceFactory(),
+        catalogModuleGitlabOrgDiscoveryEntityProvider(),
+        mockServices.rootConfig.factory({ data: config }),
+        mockServices.logger.factory(),
+        scheduler.factory,
+      ],
+    });
+
+    expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M'));
+    expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M'));
+    expect(addedProviders?.length).toEqual(1);
+    expect(runner).not.toHaveBeenCalled();
+
+    const provider = addedProviders!.pop()!;
+    expect(provider.getProviderName()).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id',
+    );
+    await provider.connect(connection);
+    expect(events.subscribed).toHaveLength(1);
+    expect(events.subscribed[0].id).toEqual(
+      'GitlabOrgDiscoveryEntityProvider:test-id',
+    );
+    expect(runner).toHaveBeenCalledTimes(1);
+  });
+});

plugins/catalog-backend-module-gitlab-org/src/catalogModuleGitlabOrgDiscoveryEntityProvider.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { loggerToWinstonLogger } from '@backstage/backend-common';
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { GitlabOrgDiscoveryEntityProvider } from '@backstage/plugin-catalog-backend-module-gitlab';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
+
+/**
+ * Registers the GitlabOrgDiscoveryEntityProvider with the catalog processing extension point.
+ *
+ * @public
+ */
+export const catalogModuleGitlabOrgDiscoveryEntityProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'gitlabOrgDiscoveryEntityProvider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          config: coreServices.rootConfig,
+          catalog: catalogProcessingExtensionPoint,
+          logger: coreServices.logger,
+          scheduler: coreServices.scheduler,
+          events: eventsServiceRef,
+        },
+        async init({ config, catalog, logger, scheduler, events }) {
+          const gitlabOrgDiscoveryEntityProvider =
+            GitlabOrgDiscoveryEntityProvider.fromConfig(config, {
+              logger: loggerToWinstonLogger(logger),
+              events,
+              scheduler,
+            });
+          catalog.addEntityProvider(gitlabOrgDiscoveryEntityProvider);
+        },
+      });
+    },
+  });

plugins/catalog-backend-module-gitlab-org/src/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The gitlab-org backend module for the catalog plugin.
+ *
+ * @packageDocumentation
+ */
+export { catalogModuleGitlabOrgDiscoveryEntityProvider as default } from './catalogModuleGitlabOrgDiscoveryEntityProvider';

plugins/catalog-backend-module-gitlab/CHANGELOG.md

@@ -385,8 +385,8 @@
 
   ref:
 
-  https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api
-  https://docs.gitlab.com/ee/api/members.html#limitations
+  <https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api>
+  <https://docs.gitlab.com/ee/api/members.html#limitations>
 
 - 890e3b5ad4: Make sure to include the error message when ingestion fails
 - 0b55f773a7: Removed some unused dependencies
@@ -417,8 +417,8 @@
 
   ref:
 
-  https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api
-  https://docs.gitlab.com/ee/api/members.html#limitations
+  <https://docs.gitlab.com/ee/user/enterprise_user/#get-users-email-addresses-through-the-api>
+  <https://docs.gitlab.com/ee/api/members.html#limitations>
 
 - 0b55f773a7: Removed some unused dependencies
 - Updated dependencies
@@ -1211,7 +1211,7 @@
 - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
 
   Please find how to configure the schedule at the config at
-  https://backstage.io/docs/integrations/gitlab/discovery
+  <https://backstage.io/docs/integrations/gitlab/discovery>
 
 - 4c9f7847e4: Updated dependency `msw` to `^0.48.0` while moving it to be a dev dependency.
 - Updated dependencies
@@ -1250,7 +1250,7 @@
 - 81cedb5033: `GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
 
   Please find how to configure the schedule at the config at
-  https://backstage.io/docs/integrations/gitlab/discovery
+  <https://backstage.io/docs/integrations/gitlab/discovery>
 
 - Updated dependencies
   - @backstage/backend-common@0.16.0-next.0

plugins/catalog-backend-module-gitlab/api-report.md

@@ -8,6 +8,7 @@ import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
 import { Config } from '@backstage/config';
 import { EntityProvider } from '@backstage/plugin-catalog-node';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
 import { GitLabIntegrationConfig } from '@backstage/integration';
 import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-node';
@@ -26,6 +27,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
     },
@@ -63,7 +65,6 @@ export type GitLabGroup = {
   name: string;
   full_path: string;
   description?: string;
-  visibility?: string;
   parent_id?: number;
 };
 
@@ -81,6 +82,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
       userTransformer?: UserTransformer;
@@ -103,6 +105,7 @@ export type GitlabProviderConfig = {
   projectPattern: RegExp;
   userPattern: RegExp;
   groupPattern: RegExp;
+  allowInherited?: boolean;
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
   skipForkedRepos?: boolean;

plugins/catalog-backend-module-gitlab/package.json

@@ -1,5 +1,21 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 {
   "name": "@backstage/plugin-catalog-backend-module-gitlab",
+  "version": "0.3.10-next.0",
   "description": "A Backstage catalog backend module that helps integrate towards GitLab",
   "version": "0.3.15",
   "main": "src/index.ts",
@@ -8,11 +24,23 @@
   "publishConfig": {
     "access": "public"
   },
+  "keywords": [
+    "backstage"
+  ],
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-gitlab"
+  },
+  "license": "Apache-2.0",
   "exports": {
     ".": "./src/index.ts",
     "./alpha": "./src/alpha.ts",
     "./package.json": "./package.json"
   },
+  "main": "src/index.ts",
+  "types": "src/index.ts",
   "typesVersions": {
     "*": {
       "alpha": [
@@ -23,26 +51,18 @@
       ]
     }
   },
-  "backstage": {
-    "role": "backend-plugin-module"
-  },
-  "homepage": "https://backstage.io",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/backstage/backstage",
-    "directory": "plugins/catalog-backend-module-gitlab"
-  },
-  "keywords": [
-    "backstage"
+  "files": [
+    "config.d.ts",
+    "dist"
   ],
   "scripts": {
-    "start": "backstage-cli package start",
     "build": "backstage-cli package build",
+    "clean": "backstage-cli package clean",
     "lint": "backstage-cli package lint",
-    "test": "backstage-cli package test",
     "prepack": "backstage-cli package prepack",
     "postpack": "backstage-cli package postpack",
-    "clean": "backstage-cli package clean"
+    "start": "backstage-cli package start",
+    "test": "backstage-cli package test"
   },
   "dependencies": {
     "@backstage/backend-common": "workspace:^",
@@ -51,7 +71,10 @@
     "@backstage/catalog-model": "workspace:^",
     "@backstage/config": "workspace:^",
     "@backstage/integration": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
     "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^",
+    "@gitbeaker/rest": "^39.25.0",
     "lodash": "^4.17.21",
     "node-fetch": "^2.6.7",
     "uuid": "^9.0.0"
@@ -59,14 +82,11 @@
   "devDependencies": {
     "@backstage/backend-test-utils": "workspace:^",
     "@backstage/cli": "workspace:^",
+    "@backstage/plugin-events-backend-test-utils": "workspace:^",
     "@types/lodash": "^4.14.151",
     "@types/uuid": "^9.0.0",
     "luxon": "^3.0.0",
     "msw": "^1.0.0"
   },
-  "files": [
-    "config.d.ts",
-    "dist"
-  ],
   "configSchema": "config.d.ts"
 }

plugins/catalog-backend-module-gitlab/src/__testUtils__/handlers.ts

@@ -0,0 +1,521 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { graphql, rest } from 'msw';
+import {
+  all_groups_response,
+  all_projects_response,
+  all_saas_users_response,
+  all_users_response,
+  apiBaseUrl,
+  apiBaseUrlSaas,
+  graphQlBaseUrl,
+  paged_endpoint,
+  saasGraphQlBaseUrl,
+  some_endpoint,
+  unhealthy_endpoint,
+  userID,
+} from './mocks';
+
+const httpHandlers = [
+  /**
+   * Project REST endpoint mocks
+   */
+
+  // fetch all projects in an instance
+  rest.get(`${apiBaseUrl}/projects`, (_, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_projects_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/projects/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  // testing non existing file
+  rest.get(
+    `${apiBaseUrl}/projects/test-group%2Ftest-repo1/repository/files/catalog-info.yaml`,
+    (_, res, ctx) => {
+      return res(ctx.status(400), ctx.json({ error: 'Not found' }));
+    },
+  ),
+
+  /**
+   * Group REST endpoint mocks
+   */
+
+  rest.get(`${apiBaseUrl}/groups`, (_req, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_groups_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/groups/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  rest.get(`${apiBaseUrlSaas}/groups/group1/members/all`, (_req, res, ctx) => {
+    return res(ctx.json(all_saas_users_response));
+  }),
+
+  /**
+   * Users REST endpoint mocks
+   */
+
+  rest.get(`${apiBaseUrl}/users`, (_req, res, ctx) => {
+    return res(ctx.set('x-next-page', ''), ctx.json(all_users_response));
+  }),
+
+  rest.get(`${apiBaseUrl}/users/${userID}`, (_, res, ctx) => {
+    return res(ctx.json(all_users_response.find(user => user.id === userID)));
+  }),
+
+  rest.get(`${apiBaseUrl}/users/42`, (_, res, ctx) => {
+    return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+  }),
+
+  /**
+   * others
+   */
+
+  // mock a 4 page response
+  rest.get(`${apiBaseUrl}${paged_endpoint}`, (req, res, ctx) => {
+    const page = req.url.searchParams.get('page');
+    const currentPage = page ? Number(page) : 1;
+    const fakePageCount = 4;
+
+    return res(
+      // set next page number header if page requested is less than count
+      ctx.set(
+        'x-next-page',
+        currentPage < fakePageCount ? String(currentPage + 1) : '',
+      ),
+      ctx.json([{ someContentOfPage: currentPage }]),
+    );
+  }),
+
+  rest.get(`${apiBaseUrl}${unhealthy_endpoint}`, (_, res, ctx) => {
+    return res(ctx.status(400), ctx.json({ error: 'some error' }));
+  }),
+
+  rest.get(`${apiBaseUrl}${some_endpoint}`, (req, res, ctx) => {
+    return res(ctx.json([{ endpoint: req.url.toString() }]));
+  }),
+];
+
+// dynamic handlers
+
+const httpGroupFindByIdDynamic = all_groups_response.map(group => {
+  return rest.get(`${apiBaseUrl}/groups/${group.id}`, (_, res, ctx) => {
+    return res(ctx.json(all_groups_response.find(g => g.id === group.id)));
+  });
+});
+
+const httpGroupListDescendantProjectsById = all_groups_response.map(group => {
+  return rest.get(
+    `${apiBaseUrl}/groups/${group.id}/projects`,
+    (_, res, ctx) => {
+      const projectsInGroup = all_projects_response.filter(p =>
+        p.path_with_namespace?.includes(group.name),
+      );
+
+      return res(ctx.json(projectsInGroup));
+    },
+  );
+});
+
+const httpGroupListDescendantProjectsByName = all_groups_response.map(group => {
+  return rest.get(
+    `${apiBaseUrl}/groups/${group.name}/projects`,
+    (_, res, ctx) => {
+      const projectsInGroup = all_projects_response.filter(p =>
+        p.path_with_namespace?.includes(group.name),
+      );
+
+      return res(ctx.json(projectsInGroup));
+    },
+  );
+});
+const httpGroupFindByNameDynamic = all_groups_response.map(group => {
+  return rest.get(`${apiBaseUrl}/groups/${group.name}`, (_, res, ctx) => {
+    return res(ctx.json(all_groups_response.find(g => g.name === group.name)));
+  });
+});
+const httpProjectFindByIdDynamic = all_projects_response.map(project => {
+  return rest.get(`${apiBaseUrl}/projects/${project.id}`, (_, res, ctx) => {
+    return res(ctx.json(all_projects_response.find(p => p.id === project.id)));
+  });
+});
+const httpProjectCatalogDynamic = all_projects_response.map(project => {
+  const path: string = project.path_with_namespace
+    ? project.path_with_namespace!.replace(/\//g, '%2F')
+    : `${project.path_with_namespace}%2F${project.name}`;
+
+  return rest.head(
+    `${apiBaseUrl}/projects/${path}/repository/files/catalog-info.yaml`,
+    (req, res, ctx) => {
+      const branch = req.url.searchParams.get('ref');
+      if (branch === project.default_branch) {
+        return res(ctx.status(200));
+      }
+      return res(ctx.status(404, 'Not Found'));
+    },
+  );
+});
+
+/**
+ * GraphQL endpoint mocks
+ */
+
+const graphqlHandlers = [
+  graphql
+    .link(graphQlBaseUrl)
+    .query('getGroupMembers', async (req, res, ctx) => {
+      const { group, relations } = req.variables;
+
+      if (group === 'group1' && relations.includes('DIRECT')) {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+      if (group === 'saas-multi-user-group') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/2',
+                      username: 'user2',
+                      publicEmail: 'user2@example.com',
+                      name: 'user2',
+                      state: 'active',
+                      webUrl: 'user2.com',
+                      avatarUrl: 'user2',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'non-existing-group' || group === '') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'multi-page' && relations.includes('DIRECT')) {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: req.variables.endCursor
+                  ? [{ user: { id: 'gid://gitlab/User/2' } }]
+                  : [{ user: { id: 'gid://gitlab/User/1' } }],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'multi-page-saas') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: req.variables.endCursor
+                  ? [
+                      {
+                        user: {
+                          id: 'gid://gitlab/User/1',
+                          username: 'user1',
+                          publicEmail: 'user1@example.com',
+                          name: 'user1',
+                          state: 'active',
+                          webUrl: 'user1.com',
+                          avatarUrl: 'user1',
+                        },
+                      },
+                    ]
+                  : [
+                      {
+                        user: {
+                          id: 'gid://gitlab/User/2',
+                          username: 'user2',
+                          publicEmail: 'user2@example.com',
+                          name: 'user2',
+                          state: 'active',
+                          webUrl: 'user2.com',
+                          avatarUrl: 'user2',
+                        },
+                      },
+                    ],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(graphQlBaseUrl)
+    .query('listDescendantGroups', async (req, res, ctx) => {
+      const { group } = req.variables;
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      if (group === 'group-with-parent') {
+        return res(
+          ctx.data({
+            group: {
+              descendantGroups: {
+                nodes: [
+                  {
+                    id: 'gid://gitlab/Group/1',
+                    name: 'group-with-parent',
+                    description: 'description1',
+                    fullPath: 'path/group-with-parent',
+                    parent: {
+                      id: '123',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+      if (group === 'root') {
+        return res(
+          ctx.data({
+            group: {
+              descendantGroups: {
+                nodes: req.variables.endCursor
+                  ? [
+                      {
+                        id: 'gid://gitlab/Group/1',
+                        name: 'group1',
+                        description: 'description1',
+                        fullPath: 'path/group1',
+                        parent: {
+                          id: '123',
+                        },
+                      },
+                    ]
+                  : [
+                      {
+                        id: 'gid://gitlab/Group/2',
+                        name: 'group2',
+                        description: 'description2',
+                        fullPath: 'path/group2',
+                        parent: {
+                          id: '123',
+                        },
+                      },
+                    ],
+                pageInfo: {
+                  endCursor: req.variables.endCursor ? 'end' : 'next',
+                  hasNextPage: !req.variables.endCursor,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === 'non-existing-group') {
+        return res(
+          ctx.data({
+            group: {},
+          }),
+        );
+      }
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(saasGraphQlBaseUrl)
+    .query('getGroupMembers', async (req, res, ctx) => {
+      const { group } = req.variables;
+
+      if (group === 'saas-multi-user-group') {
+        return res(
+          ctx.data({
+            group: {
+              groupMembers: {
+                nodes: [
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/1',
+                      username: 'user1',
+                      publicEmail: 'user1@example.com',
+                      name: 'user1',
+                      state: 'active',
+                      webUrl: 'user1.com',
+                      avatarUrl: 'user1',
+                    },
+                  },
+                  {
+                    user: {
+                      id: 'gid://gitlab/User/2',
+                      username: 'user2',
+                      publicEmail: 'user2@example.com',
+                      name: 'user2',
+                      state: 'active',
+                      webUrl: 'user2.com',
+                      avatarUrl: 'user2',
+                    },
+                  },
+                ],
+                pageInfo: {
+                  endCursor: 'end',
+                  hasNextPage: false,
+                },
+              },
+            },
+          }),
+        );
+      }
+
+      if (group === '') {
+        return res(ctx.data({}));
+      }
+
+      if (group === 'error-group') {
+        return res(
+          ctx.errors([
+            { message: 'Unexpected end of document', locations: [] },
+          ]),
+        );
+      }
+
+      return res(ctx.status(200));
+    }),
+
+  graphql
+    .link(saasGraphQlBaseUrl)
+    .query('listDescendantGroups', async (_, res, ctx) => {
+      return res(
+        ctx.data({
+          group: {
+            descendantGroups: {
+              nodes: [
+                {
+                  id: 'gid://gitlab/Group/456',
+                  name: 'group1',
+                  description: 'Group1',
+                  fullPath: 'group1/group1',
+                  parent: {
+                    id: 'gid://gitlab/Group/123',
+                  },
+                },
+              ],
+              pageInfo: {
+                endCursor: 'end',
+                hasNextPage: false,
+              },
+            },
+          },
+        }),
+      );
+    }),
+];
+
+export const handlers = [
+  ...httpHandlers,
+  ...httpProjectFindByIdDynamic,
+  ...httpProjectCatalogDynamic,
+  ...httpGroupFindByIdDynamic,
+  ...httpGroupFindByNameDynamic,
+  ...httpGroupListDescendantProjectsById,
+  ...httpGroupListDescendantProjectsByName,
+  ...graphqlHandlers,
+];

plugins/catalog-backend-module-gitlab/src/lib/client.ts

@@ -24,6 +24,7 @@ import {
   GitLabDescendantGroupsResponse,
   GitLabGroup,
   GitLabGroupMembersResponse,
+  GitLabProject,
   GitLabUser,
   PagedResponse,
 } from './types';
@@ -80,6 +81,39 @@ export class GitLabClient {
     return this.pagedRequest(`/projects`, options);
   }
 
+  async getProjectById(
+    projectId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabProject> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(
+      `/projects/${projectId}`,
+      options,
+    );
+
+    return response;
+  }
+
+  async getGroupById(
+    groupId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabGroup> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(`/groups/${groupId}`, options);
+
+    return response;
+  }
+
+  async getUserById(
+    userId: number,
+    options?: CommonListOptions,
+  ): Promise<GitLabUser> {
+    // Make the request to the GitLab API
+    const response = await this.nonPagedRequest(`/users/${userId}`, options);
+
+    return response;
+  }
+
   async listUsers(
     options?: UserListOptions,
   ): Promise<PagedResponse<GitLabUser>> {
@@ -139,7 +173,6 @@ export class GitLabClient {
                       name
                       description
                       fullPath
-                      visibility
                       parent {
                         id
                       }
@@ -174,7 +207,6 @@ export class GitLabClient {
           name: groupItem.name,
           description: groupItem.description,
           full_path: groupItem.fullPath,
-          visibility: groupItem.visibility,
           parent_id: Number(
             groupItem.parent.id.replace(/^gid:\/\/gitlab\/Group\//, ''),
           ),
@@ -325,9 +357,13 @@ export class GitLabClient {
     options?: CommonListOptions,
   ): Promise<PagedResponse<T>> {
     const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
+
     for (const key in options) {
-      if (options[key] !== undefined && options[key] !== '') {
-        request.searchParams.append(key, options[key]!.toString());
+      if (options.hasOwnProperty(key)) {
+        const value = options[key];
+        if (value !== undefined && value !== '') {
+          request.searchParams.append(key, value.toString());
+        }
       }
     }
 
@@ -336,6 +372,7 @@ export class GitLabClient {
       request.toString(),
       getGitLabRequestOptions(this.config),
     );
+
     if (!response.ok) {
       throw new Error(
         `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
@@ -343,6 +380,7 @@ export class GitLabClient {
         } - ${response.statusText}`,
       );
     }
+
     return response.json().then(items => {
       const nextPage = response.headers.get('x-next-page');
 
@@ -352,6 +390,37 @@ export class GitLabClient {
       } as PagedResponse<any>;
     });
   }
+
+  async nonPagedRequest<T = any>(
+    endpoint: string,
+    options?: CommonListOptions,
+  ): Promise<T> {
+    const request = new URL(`${this.config.apiBaseUrl}${endpoint}`);
+
+    for (const key in options) {
+      if (options.hasOwnProperty(key)) {
+        const value = options[key];
+        if (value !== undefined && value !== '') {
+          request.searchParams.append(key, value.toString());
+        }
+      }
+    }
+
+    const response = await fetch(
+      request.toString(),
+      getGitLabRequestOptions(this.config),
+    );
+
+    if (!response.ok) {
+      throw new Error(
+        `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
+          response.status
+        } - ${response.statusText}`,
+      );
+    }
+
+    return response.json();
+  }
 }
 
 /**

plugins/catalog-backend-module-gitlab/src/lib/defaultTransformers.ts

@@ -49,10 +49,6 @@ export function defaultGroupEntitiesTransformer(
     const annotations: { [annotationName: string]: string } = {};
 
     annotations[`${options.providerConfig.host}/team-path`] = group.full_path;
-    if (group.visibility !== undefined) {
-      annotations[`${options.providerConfig.host}/visibility`] =
-        group.visibility;
-    }
 
     const entity: GroupEntity = {
       apiVersion: 'backstage.io/v1alpha1',

plugins/catalog-backend-module-gitlab/src/lib/index.ts

@@ -14,14 +14,15 @@
  * limitations under the License.
  */
 
+export { readGitlabConfigs } from '../providers/config';
 export { GitLabClient, paginated } from './client';
 export type {
-  GitLabUser,
   GitLabGroup,
   GitLabGroupSamlIdentity,
   GitLabProject,
-  GitlabProviderConfig,
+  GitLabUser,
   GitlabGroupDescription,
+  GitlabProviderConfig,
   GroupNameTransformer,
   GroupNameTransformerOptions,
   GroupTransformer,
@@ -29,4 +30,3 @@ export type {
   UserTransformer,
   UserTransformerOptions,
 } from './types';
-export { readGitlabConfigs } from '../providers/config';

plugins/catalog-backend-module-gitlab/src/lib/types.ts

@@ -34,6 +34,9 @@ export type GitlabProjectForkedFrom = {
 
 export type GitLabProject = {
   id: number;
+  description?: string;
+  name?: string;
+  path?: string;
   default_branch?: string;
   archived: boolean;
   last_activity_at: string;
@@ -76,7 +79,6 @@ export type GitLabGroup = {
   name: string;
   full_path: string;
   description?: string;
-  visibility?: string;
   parent_id?: number;
 };
 
@@ -116,7 +118,6 @@ export type GitLabDescendantGroupsResponse = {
             name: string;
             description: string;
             fullPath: string;
-            visibility: string;
             parent: {
               id: string;
             };
@@ -180,6 +181,11 @@ export type GitlabProviderConfig = {
    */
   groupPattern: RegExp;
 
+  /**
+   * If true, the provider will also ingest add inherited users to the ingested groups
+   */
+  allowInherited?: boolean;
+
   orgEnabled?: boolean;
   schedule?: TaskScheduleDefinition;
   /**
@@ -242,3 +248,72 @@ export interface GroupTransformerOptions {
   providerConfig: GitlabProviderConfig;
   groupNameTransformer: GroupNameTransformer;
 }
+
+/**
+ * Represents the schema for system hook events related to groups.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseGroupEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  name: string;
+  path: string;
+  full_path: string;
+  group_id: number;
+};
+
+/**
+ * Represents the schema for system hook events related to users.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseUserEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  email: string;
+  name: string;
+  username: string;
+  user_id: number;
+};
+
+/**
+ * Represents the schema for system hook events related to user memberships.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseMembershipEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  group_name: string;
+  group_path: string;
+  group_id: number;
+  user_username: string;
+  user_email: string;
+  user_name: string;
+  user_id: number;
+  group_access: string;
+};
+
+/**
+ * Represents the schema for system hook events related to projects.
+ * https://docs.gitlab.com/ee/administration/system_hooks.html
+ *
+ * @public
+ */
+export type SystemHookBaseProjectEventsSchema = {
+  created_at: string;
+  updated_at: string;
+  event_name: string;
+  name: string;
+  owner_email: string;
+  owner_name: string;
+  owners: { name: string; email: string }[];
+  path: string;
+  path_with_namespace: string;
+  project_id: number;
+  project_visibility: string;
+};

plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.test.ts

@@ -14,15 +14,27 @@
  * limitations under the License.
  */
 
+import { createServiceFactory } from '@backstage/backend-plugin-api';
 import { TaskScheduleDefinition } from '@backstage/backend-tasks';
 import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
 import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { TestEventsService } from '@backstage/plugin-events-backend-test-utils';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
 import { Duration } from 'luxon';
-import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider';
 import { GitlabDiscoveryEntityProvider } from '../providers';
+import { catalogModuleGitlabDiscoveryEntityProvider } from './catalogModuleGitlabDiscoveryEntityProvider';
 
 describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
   it('should register provider at the catalog extension point', async () => {
+    const events = new TestEventsService();
+    const eventsServiceFactory = createServiceFactory({
+      service: eventsServiceRef,
+      deps: {},
+      async factory({}) {
+        return events;
+      },
+    });
     let addedProviders: Array<GitlabDiscoveryEntityProvider> | undefined;
     let usedSchedule: TaskScheduleDefinition | undefined;
 
@@ -31,6 +43,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
         addedProviders = providers;
       },
     };
+    const connection = jest.fn() as unknown as EntityProviderConnection;
     const runner = jest.fn();
     const scheduler = mockServices.scheduler.mock({
       createScheduledTaskRunner(schedule) {
@@ -68,6 +81,7 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
     await startTestBackend({
       extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
       features: [
+        eventsServiceFactory(),
         catalogModuleGitlabDiscoveryEntityProvider(),
         mockServices.rootConfig.factory({ data: config }),
         mockServices.logger.factory(),
@@ -78,9 +92,17 @@ describe('catalogModuleGitlabDiscoveryEntityProvider', () => {
     expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M'));
     expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M'));
     expect(addedProviders?.length).toEqual(1);
-    expect(addedProviders?.pop()?.getProviderName()).toEqual(
+    expect(runner).not.toHaveBeenCalled();
+
+    const provider = addedProviders!.pop()!;
+    expect(provider.getProviderName()).toEqual(
       'GitlabDiscoveryEntityProvider:test-id',
     );
-    expect(runner).not.toHaveBeenCalled();
+    await provider.connect(connection);
+    expect(events.subscribed).toHaveLength(1);
+    expect(events.subscribed[0].id).toEqual(
+      'GitlabDiscoveryEntityProvider:test-id',
+    );
+    expect(runner).toHaveBeenCalledTimes(1);
   });
 });

plugins/catalog-backend-module-gitlab/src/module/catalogModuleGitlabDiscoveryEntityProvider.ts

@@ -19,6 +19,7 @@ import {
   createBackendModule,
 } from '@backstage/backend-plugin-api';
 import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
 import { GitlabDiscoveryEntityProvider } from '../providers';
 
 /**
@@ -26,6 +27,7 @@ import { GitlabDiscoveryEntityProvider } from '../providers';
  *
  * @alpha
  */
+
 export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({
   pluginId: 'catalog',
   moduleId: 'gitlab-discovery-entity-provider',
@@ -36,14 +38,16 @@ export const catalogModuleGitlabDiscoveryEntityProvider = createBackendModule({
         catalog: catalogProcessingExtensionPoint,
         logger: coreServices.logger,
         scheduler: coreServices.scheduler,
+        events: eventsServiceRef,
       },
-      async init({ config, catalog, logger, scheduler }) {
-        catalog.addEntityProvider(
+      async init({ config, catalog, logger, scheduler, events }) {
+        const gitlabDiscoveryEntityProvider =
           GitlabDiscoveryEntityProvider.fromConfig(config, {
             logger,
+            events,
             scheduler,
-          }),
-        );
+          });
+        catalog.addEntityProvider(gitlabDiscoveryEntityProvider);
       },
     });
   },

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -23,10 +23,16 @@ import {
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
-import { rest } from 'msw';
+import { DefaultEventsService } from '@backstage/plugin-events-node';
 import { setupServer } from 'msw/node';
+import { handlers } from '../__testUtils__/handlers';
+import * as mock from '../__testUtils__/mocks';
 import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider';
 
+const server = setupServer(...handlers);
+setupRequestMockHandlers(server);
+afterEach(() => jest.resetAllMocks());
+
 class PersistingTaskRunner implements TaskRunner {
   private tasks: TaskInvocationDefinition[] = [];
 
@@ -42,13 +48,8 @@ class PersistingTaskRunner implements TaskRunner {
 
 const logger = getVoidLogger();
 
-const server = setupServer();
-
-describe('GitlabDiscoveryEntityProvider', () => {
-  setupRequestMockHandlers(server);
-  afterEach(() => jest.resetAllMocks());
-
-  it('no provider config', () => {
+describe('GitlabDiscoveryEntityProvider - configuration', () => {
+  it('should not instantiate providers when no config found', () => {
     const schedule = new PersistingTaskRunner();
     const config = new ConfigReader({});
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
@@ -59,29 +60,46 @@ describe('GitlabDiscoveryEntityProvider', () => {
     expect(providers).toHaveLength(0);
   });
 
-  it('single simple discovery config', () => {
+  it('should fail without schedule nor scheduler', () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+
+    expect(() =>
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+      }),
+    ).toThrow('Either schedule or scheduler must be provided');
+  });
+
+  it('should fail with scheduler but no schedule config', () => {
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => jest.fn(),
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader(mock.config_no_schedule_integration);
+
+    expect(() =>
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+        scheduler,
+      }),
+    ).toThrow(
+      'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id',
+    );
+  });
+  it('should throw error when no matching GitLab integration config found', () => {
     const schedule = new PersistingTaskRunner();
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_non_gitlab_host);
+
+    expect(() => {
+      GitlabDiscoveryEntityProvider.fromConfig(config, {
+        logger,
+        schedule,
+      });
+    }).toThrow('No gitlab integration found that matches host example.com');
+  });
+
+  it('should instantiate provider with single simple discovery config', () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader(mock.config_single_integration_branch);
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
       schedule,
@@ -93,33 +111,9 @@ describe('GitlabDiscoveryEntityProvider', () => {
     );
   });
 
-  it('multiple discovery configs', () => {
+  it('should instantiate providers when multiple discovery configs', () => {
     const schedule = new PersistingTaskRunner();
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-            'second-test': {
-              host: 'test-gitlab',
-              group: 'second-group',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_double_integration);
     const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
       schedule,
@@ -133,29 +127,10 @@ describe('GitlabDiscoveryEntityProvider', () => {
       'GitlabDiscoveryEntityProvider:second-test',
     );
   });
-
-  it('apply full update on scheduled execution', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
+});
+describe('GitlabDiscoveryEntityProvider - refresh', () => {
+  it('should apply full update on scheduled execution', async () => {
+    const config = new ConfigReader(mock.config_no_org_integration);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -169,92 +144,28 @@ describe('GitlabDiscoveryEntityProvider', () => {
       'GitlabDiscoveryEntityProvider:test-id',
     );
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/groups/test-group/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
-
     await provider.connect(entityProviderConnection);
 
     const taskDef = schedule.getTasks()[0];
     expect(taskDef.id).toEqual('GitlabDiscoveryEntityProvider:test-id:refresh');
     await (taskDef.fn as () => Promise<void>)();
 
-    const url = `https://api.gitlab.example/test-group/test-repo/-/blob/master/catalog-info.yaml`;
-    const expectedEntities = [
-      {
-        entity: {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Location',
-          metadata: {
-            annotations: {
-              'backstage.io/managed-by-location': `url:${url}`,
-              'backstage.io/managed-by-origin-location': `url:${url}`,
-            },
-            name: 'generated-cd37bf72a2fe92603f4255d9f49c6c1ead746a48',
-          },
-          spec: {
-            presence: 'optional',
-            target: `${url}`,
-            type: 'url',
-          },
-        },
-        locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-      },
-    ];
-
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: expectedEntities,
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
   it('should filter found projects based on a provided project pattern', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              projectPattern: 'john/',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(
+      mock.config_single_integration_project_pattern,
+    );
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -265,50 +176,10 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
+    const projectPattern =
+      mock.config_single_integration_project_pattern.catalog.providers.gitlab[
+        'test-id'
+      ].projectPattern;
 
     await provider.connect(entityProviderConnection);
 
@@ -316,55 +187,20 @@ describe('GitlabDiscoveryEntityProvider', () => {
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              },
-              name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(projectPattern) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
   it('should filter fork projects', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              skipForkedRepos: true,
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_single_integration_skip_forks);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -375,216 +211,26 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-              forked_from_project: {
-                id: 13083,
-              },
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'master') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
-
     await provider.connect(entityProviderConnection);
 
     await provider.refresh(logger);
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              },
-              name: 'generated-2045212e5b3e9e6bacf51cec709e362282e3cda9',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/master/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('forked') &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
     });
   });
 
-  it('fail without schedule and scheduler', () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
-
-    expect(() =>
-      GitlabDiscoveryEntityProvider.fromConfig(config, {
-        logger,
-      }),
-    ).toThrow('Either schedule or scheduler must be provided');
-  });
-
-  it('fail with scheduler but no schedule config', () => {
-    const scheduler = {
-      createScheduledTaskRunner: (_: any) => jest.fn(),
-    } as unknown as PluginTaskScheduler;
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-            },
-          },
-        },
-      },
-    });
-
-    expect(() =>
-      GitlabDiscoveryEntityProvider.fromConfig(config, {
-        logger,
-        scheduler,
-      }),
-    ).toThrow(
-      'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id',
-    );
-  });
-
-  it('single simple provider config with schedule in config', async () => {
-    const schedule = new PersistingTaskRunner();
-    const scheduler = {
-      createScheduledTaskRunner: (_: any) => schedule,
-    } as unknown as PluginTaskScheduler;
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              group: 'test-group',
-              schedule: {
-                frequency: 'PT30M',
-                timeout: 'PT3M',
-              },
-            },
-          },
-        },
-      },
-    });
-    const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
-      logger,
-      scheduler,
-    });
-
-    expect(providers).toHaveLength(1);
-    expect(providers[0].getProviderName()).toEqual(
-      'GitlabDiscoveryEntityProvider:test-id',
-    );
-  });
-
   it('should filter found projects based on the branch', async () => {
-    const config = new ConfigReader({
-      integrations: {
-        gitlab: [
-          {
-            host: 'test-gitlab',
-            apiBaseUrl: 'https://api.gitlab.example/api/v4',
-            token: '1234',
-          },
-        ],
-      },
-      catalog: {
-        providers: {
-          gitlab: {
-            'test-id': {
-              host: 'test-gitlab',
-              branch: 'test',
-            },
-          },
-        },
-      },
-    });
+    const config = new ConfigReader(mock.config_single_integration_branch);
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
@@ -595,47 +241,9 @@ describe('GitlabDiscoveryEntityProvider', () => {
       schedule,
     })[0];
 
-    server.use(
-      rest.get(
-        `https://api.gitlab.example/api/v4/projects`,
-        (_req, res, ctx) => {
-          const response = [
-            {
-              id: 123,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/test-group/test-repo',
-              path_with_namespace: 'test-group/test-repo',
-            },
-            {
-              id: 124,
-              default_branch: 'master',
-              archived: false,
-              last_activity_at: new Date().toString(),
-              web_url: 'https://api.gitlab.example/john/example',
-              path_with_namespace: 'john/example',
-            },
-          ];
-          return res(ctx.json(response));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/test-group%2Ftest-repo/repository/files/catalog-info.yaml',
-        (_, res, ctx) => {
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-      rest.head(
-        'https://api.gitlab.example/api/v4/projects/john%2Fexample/repository/files/catalog-info.yaml',
-        (req, res, ctx) => {
-          if (req.url.searchParams.get('ref') === 'test') {
-            return res(ctx.status(200));
-          }
-          return res(ctx.status(404, 'Not Found'));
-        },
-      ),
-    );
+    const configured_branch =
+      mock.config_single_integration_branch.catalog.providers.gitlab['test-id']
+        .branch;
 
     await provider.connect(entityProviderConnection);
 
@@ -643,30 +251,229 @@ describe('GitlabDiscoveryEntityProvider', () => {
 
     expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
       type: 'full',
-      entities: [
-        {
-          entity: {
-            apiVersion: 'backstage.io/v1alpha1',
-            kind: 'Location',
-            metadata: {
-              annotations: {
-                'backstage.io/managed-by-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-                'backstage.io/managed-by-origin-location':
-                  'url:https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-              },
-              name: 'generated-232185d858fee049986d202c10316d634e76a3d1',
-            },
-            spec: {
-              presence: 'optional',
-              target:
-                'https://api.gitlab.example/john/example/-/blob/test/catalog-info.yaml',
-              type: 'url',
-            },
-          },
-          locationKey: 'GitlabDiscoveryEntityProvider:test-id',
-        },
-      ],
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(configured_branch) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
+    });
+  });
+
+  it('should only include projects with fallback branch', async () => {
+    const config = new ConfigReader(mock.config_fallbackBranch_branch);
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const configured_branch =
+      mock.config_fallbackBranch_branch.catalog.providers.gitlab['test-id']
+        .fallbackBranch;
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities.filter(
+        entity =>
+          entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes(configured_branch) &&
+          !entity.entity.metadata.annotations[
+            'backstage.io/managed-by-location'
+          ].includes('awesome'),
+      ),
+    });
+  });
+
+  it('should ignore projects outside group scope', async () => {
+    const config = new ConfigReader(mock.config_single_integration_group);
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const configured_group =
+      mock.config_single_integration_group.catalog.providers.gitlab['test-id']
+        .group;
+
+    await provider.connect(entityProviderConnection);
+
+    await provider.refresh(logger);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: mock.expected_location_entities.filter(entity =>
+        entity.entity.metadata.annotations[
+          'backstage.io/managed-by-location'
+        ].includes(configured_group),
+      ),
     });
   });
 });
+describe('GitlabDiscoveryEntityProvider - events', () => {
+  it('should ignore push event if project is forked', async () => {
+    const config = new ConfigReader(mock.config_single_integration_skip_forks);
+
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event_forked);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it(`should skip refresh and mutation when project pattern doesn't match`, async () => {
+    const config = new ConfigReader(mock.config_unmatched_project_integration);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('should ignore projects outside group scope', async () => {
+    const config = new ConfigReader(mock.config_single_integration_group);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event_unmatched_group);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+  it('should apply delta mutations on added files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_add_event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: mock.expected_added_location_entities,
+      removed: [],
+    });
+  });
+
+  it('should apply delta mutations on removed files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    await events.publish(mock.push_remove_event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: [],
+      removed: mock.expected_removed_location_entities,
+    });
+  });
+
+  it('should call refresh on added files from push event', async () => {
+    const config = new ConfigReader(mock.config_single_integration_branch);
+    const schedule = new PersistingTaskRunner();
+    const events = DefaultEventsService.create({ logger });
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      events,
+    })[0];
+
+    await provider.connect(entityProviderConnection);
+
+    const url = `https://example.com/group1/test-repo1`;
+
+    await events.publish(mock.push_modif_event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
+      keys: [
+        `url:${url}/-/tree/main/catalog-info.yaml`,
+        `url:${url}/-/blob/main/catalog-info.yaml`,
+      ],
+    });
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+  // EventSupportChange: stop add tests >>>
+});

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts

@@ -17,12 +17,15 @@
 import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import { GitLabIntegration, ScmIntegrations } from '@backstage/integration';
+import { LocationSpec } from '@backstage/plugin-catalog-common';
 import {
+  DeferredEntity,
   EntityProvider,
   EntityProviderConnection,
-  LocationSpec,
   locationSpecToLocationEntity,
 } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
+import { WebhookProjectSchema, WebhookPushEventSchema } from '@gitbeaker/rest';
 import * as uuid from 'uuid';
 import {
   GitLabClient,
@@ -33,26 +36,37 @@ import {
 } from '../lib';
 import { LoggerService } from '@backstage/backend-plugin-api';
 
+import * as path from 'path';
+
+const TOPIC_REPO_PUSH = 'gitlab.push';
+
 type Result = {
   scanned: number;
   matches: GitLabProject[];
 };
 
 /**
- * Discovers entity definition files in the groups of a Gitlab instance.
+ * Discovers catalog files located in your GitLab instance.
+ * The provider will search your GitLab instance's projects and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
  * @public
  */
+// <<< EventSupportChange: implemented EventSubscriber interface
 export class GitlabDiscoveryEntityProvider implements EntityProvider {
   private readonly config: GitlabProviderConfig;
   private readonly integration: GitLabIntegration;
   private readonly logger: LoggerService;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
+  private readonly events?: EventsService;
 
   static fromConfig(
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
     },
@@ -92,13 +106,20 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
         }),
       );
     });
+
     return providers;
   }
 
+  /**
+   * Constructs a GitlabDiscoveryEntityProvider instance.
+   *
+   * @param options - Configuration options including config, integration, logger, and taskRunner.
+   */
   private constructor(options: {
     config: GitlabProviderConfig;
     integration: GitLabIntegration;
     logger: LoggerService;
+    events?: EventsService;
     taskRunner: TaskRunner;
   }) {
     this.config = options.config;
@@ -107,6 +128,7 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       target: this.getProviderName(),
     });
     this.scheduleFn = this.createScheduleFn(options.taskRunner);
+    this.events = options.events;
   }
 
   getProviderName(): string {
@@ -116,8 +138,28 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
   async connect(connection: EntityProviderConnection): Promise<void> {
     this.connection = connection;
     await this.scheduleFn();
+
+    if (this.events) {
+      await this.events.subscribe({
+        id: this.getProviderName(),
+        topics: [TOPIC_REPO_PUSH],
+        onEvent: async params => {
+          if (params.topic !== TOPIC_REPO_PUSH) {
+            return;
+          }
+
+          await this.onRepoPush(params.eventPayload as WebhookPushEventSchema);
+        },
+      });
+    }
   }
 
+  /**
+   * Creates a scheduled task runner for refreshing the entity provider.
+   *
+   * @param taskRunner - The task runner instance.
+   * @returns The scheduled function.
+   */
   private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
     return async () => {
       const taskId = `${this.getProviderName()}:refresh`;
@@ -143,13 +185,17 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     };
   }
 
+  /**
+   * Performs a full scan on the GitLab instance searching for locations to be ingested
+   *
+   * @param logger - The logger instance for logging.
+   */
   async refresh(logger: LoggerService): Promise<void> {
     if (!this.connection) {
       throw new Error(
         `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
       );
     }
-
     const client = new GitLabClient({
       config: this.integration.config,
       logger: logger,
@@ -171,38 +217,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
     };
 
     for await (const project of projects) {
-      if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) {
-        continue;
-      }
-
-      res.scanned++;
-
-      if (
-        this.config.skipForkedRepos &&
-        project.hasOwnProperty('forked_from_project')
-      ) {
-        continue;
-      }
-
-      if (
-        !this.config.branch &&
-        this.config.fallbackBranch === '*' &&
-        project.default_branch === undefined
-      ) {
-        continue;
-      }
-
-      const project_branch =
-        this.config.branch ??
-        project.default_branch ??
-        this.config.fallbackBranch;
-
-      const projectHasFile: boolean = await client.hasFile(
-        project.path_with_namespace ?? '',
-        project_branch,
-        this.config.catalogFile,
-      );
-      if (projectHasFile) {
+      if (await this.shouldProcessProject(project, client)) {
+        res.scanned++;
         res.matches.push(project);
       }
     }
@@ -222,10 +238,255 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
       this.config.branch ??
       project.default_branch ??
       this.config.fallbackBranch;
+
     return {
       type: 'url',
       target: `${project.web_url}/-/blob/${project_branch}/${this.config.catalogFile}`,
       presence: 'optional',
     };
   }
+
+  /**
+   * Handles the "gitlab.push" event.
+   *
+   * @param event - The push event payload.
+   */
+  private async onRepoPush(event: WebhookPushEventSchema): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+    this.logger.info(
+      `Received push event for ${event.project.path_with_namespace}`,
+    );
+
+    const client = new GitLabClient({
+      config: this.integration.config,
+      logger: this.logger,
+    });
+
+    const project = await client.getProjectById(event.project_id);
+
+    if (!project) {
+      this.logger.debug(
+        `Ignoring push event for ${event.project.path_with_namespace}`,
+      );
+
+      return;
+    }
+
+    if (!(await this.shouldProcessProject(project, client))) {
+      this.logger.debug(`Skipping event ${event.project.path_with_namespace}`);
+      return;
+    }
+
+    // Get array of added, removed or modified files from the push event
+    const added = this.getFilesMatchingConfig(
+      event,
+      'added',
+      this.config.catalogFile,
+    );
+    const removed = this.getFilesMatchingConfig(
+      event,
+      'removed',
+      this.config.catalogFile,
+    );
+    const modified = this.getFilesMatchingConfig(
+      event,
+      'modified',
+      this.config.catalogFile,
+    );
+
+    // Modified files will be scheduled to a refresh
+    const addedEntities = this.createLocationSpecCommitedFiles(
+      event.project,
+      added,
+    );
+
+    const removedEntities = this.createLocationSpecCommitedFiles(
+      event.project,
+      removed,
+    );
+
+    if (addedEntities.length > 0 || removedEntities.length > 0) {
+      await this.connection.applyMutation({
+        type: 'delta',
+        added: this.toDeferredEntities(
+          addedEntities.map(entity => entity.target),
+        ),
+        removed: this.toDeferredEntities(
+          removedEntities.map(entity => entity.target),
+        ),
+      });
+    }
+
+    if (modified.length > 0) {
+      const projectBranch =
+        this.config.branch ??
+        event.project.default_branch ??
+        this.config.fallbackBranch;
+
+      // scheduling a refresh to both tree and blob (https://git-scm.com/book/en/v2/Git-Internals-Git-Objects)
+      await this.connection.refresh({
+        keys: [
+          ...modified.map(
+            filePath =>
+              `url:${event.project.web_url}/-/tree/${projectBranch}/${filePath}`,
+          ),
+          ...modified.map(
+            filePath =>
+              `url:${event.project.web_url}/-/blob/${projectBranch}/${filePath}`,
+          ),
+        ],
+      });
+    }
+
+    this.logger.info(
+      `Processed GitLab push event from ${event.project.web_url}: added ${added.length} - removed ${removed.length} - modified ${modified.length}`,
+    );
+  }
+
+  /**
+   * Gets files matching the specified commit action and catalog file name.
+   *
+   * @param event - The push event payload.
+   * @param action - The action type ('added', 'removed', or 'modified').
+   * @param catalogFile - The catalog file name.
+   * @returns An array of file paths.
+   */
+  private getFilesMatchingConfig(
+    event: WebhookPushEventSchema,
+    action: 'added' | 'removed' | 'modified',
+    catalogFile: string,
+  ): string[] {
+    if (!event.commits) {
+      return [];
+    }
+
+    const matchingFiles = event.commits.flatMap((element: any) =>
+      element[action].filter(
+        (file: string) => path.basename(file) === catalogFile,
+      ),
+    );
+
+    if (matchingFiles.length === 0) {
+      this.logger.debug(
+        `No files matching '${catalogFile}' found in the commits.`,
+      );
+    }
+
+    return matchingFiles;
+  }
+
+  /**
+   * Creates Backstage location specs for committed files.
+   *
+   * @param project - The GitLab project information.
+   * @param addedFiles - The array of added file paths.
+   * @returns An array of location specs.
+   */
+  private createLocationSpecCommitedFiles(
+    project: WebhookProjectSchema,
+    addedFiles: string[],
+  ): LocationSpec[] {
+    const projectBranch =
+      this.config.branch ??
+      project.default_branch ??
+      this.config.fallbackBranch;
+
+    // Filter added files that match the catalog file pattern
+    const matchingFiles = addedFiles.filter(
+      file => path.basename(file) === this.config.catalogFile,
+    );
+
+    // Create a location spec for each matching file
+    const locationSpecs: LocationSpec[] = matchingFiles.map(file => ({
+      type: 'url',
+      target: `${project.web_url}/-/blob/${projectBranch}/${file}`,
+      presence: 'optional',
+    }));
+
+    return locationSpecs;
+  }
+
+  /**
+   * Converts a target URL to a LocationSpec object.
+   *
+   * @param {string} target - The target URL to be converted.
+   * @returns {LocationSpec} The LocationSpec object representing the URL.
+   */
+  private toLocationSpec(target: string): LocationSpec {
+    return {
+      type: 'url',
+      target: target,
+      presence: 'optional',
+    };
+  }
+
+  private toDeferredEntities(targets: string[]): DeferredEntity[] {
+    return targets
+      .map(target => {
+        const location = this.toLocationSpec(target);
+
+        return locationSpecToLocationEntity({ location });
+      })
+      .map(entity => {
+        return {
+          locationKey: this.getProviderName(),
+          entity: entity,
+        };
+      });
+  }
+
+  private async shouldProcessProject(
+    project: GitLabProject,
+    client: GitLabClient,
+  ): Promise<boolean> {
+    if (!this.config.projectPattern.test(project.path_with_namespace ?? '')) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it does not match the project pattern ${this.config.projectPattern}.`,
+      );
+      return false;
+    }
+
+    if (
+      this.config.group &&
+      !project.path_with_namespace!.startsWith(`${this.config.group}/`)
+    ) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it does not match the group pattern ${this.config.group}.`,
+      );
+      return false;
+    }
+
+    if (
+      this.config.skipForkedRepos &&
+      project.hasOwnProperty('forked_from_project')
+    ) {
+      this.logger.debug(
+        `Skipping project ${project.path_with_namespace} as it is a forked project.`,
+      );
+      return false;
+    }
+
+    const customFallbackBranch =
+      this.config.fallbackBranch !== 'master'
+        ? this.config.fallbackBranch
+        : undefined;
+
+    const project_branch =
+      this.config.branch ??
+      customFallbackBranch ??
+      project.default_branch ??
+      this.config.fallbackBranch;
+
+    const hasFile = await client.hasFile(
+      project.path_with_namespace ?? '',
+      project_branch,
+      this.config.catalogFile,
+    );
+
+    return hasFile;
+  }
 }

plugins/catalog-backend-module-gitlab/src/providers/GitlabOrgDiscoveryEntityProvider.ts

@@ -25,6 +25,7 @@ import {
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-node';
+import { EventsService } from '@backstage/plugin-events-node';
 import { merge } from 'lodash';
 import * as uuid from 'uuid';
 import { LoggerService } from '@backstage/backend-plugin-api';
@@ -35,21 +36,24 @@ import {
   paginated,
   readGitlabConfigs,
 } from '../lib';
-import {
-  GitLabGroup,
-  GitLabUser,
-  GroupNameTransformer,
-  GroupTransformer as GroupEntitiesTransformer,
-  PagedResponse,
-  UserTransformer,
-} from '../lib/types';
 import {
   defaultGroupEntitiesTransformer,
   defaultGroupNameTransformer,
   defaultUserTransformer,
 } from '../lib/defaultTransformers';
+import {
+  GitLabGroup,
+  GitLabUser,
+  GroupTransformer as GroupEntitiesTransformer,
+  GroupNameTransformer,
+  PagedResponse,
+  SystemHookBaseGroupEventsSchema,
+  SystemHookBaseMembershipEventsSchema,
+  SystemHookBaseUserEventsSchema,
+  UserTransformer,
+} from '../lib/types';
 
-type Result = {
+type UserResult = {
   scanned: number;
   matches: GitLabUser[];
 };
@@ -59,6 +63,36 @@ type GroupResult = {
   matches: GitLabGroup[];
 };
 
+type SystemHookGroupCreateOrDestroyEventSchema =
+  SystemHookBaseGroupEventsSchema & {
+    event_name: 'group_create' | 'group_destroy';
+  };
+
+type SystemHookGroupRenameEventSchema = SystemHookBaseGroupEventsSchema & {
+  event_name: 'group_rename';
+  old_path: string;
+  old_full_path: string;
+};
+
+type SystemHookUserCreateOrDestroyEventSchema =
+  SystemHookBaseUserEventsSchema & {
+    event_name: 'user_create' | 'user_destroy';
+  };
+
+type SystemHookCreateOrDestroyMembershipEventsSchema =
+  SystemHookBaseMembershipEventsSchema & {
+    event_name: 'user_add_to_group' | 'user_remove_from_group';
+  };
+
+// System level events
+const TOPIC_GROUP_CREATE = 'gitlab.group_create';
+const TOPIC_GROUP_DESTROY = 'gitlab.group_destroy';
+const TOPIC_GROUP_RENAME = 'gitlab.group_rename';
+const TOPIC_USER_CREATE = 'gitlab.user_create';
+const TOPIC_USER_DESTROY = 'gitlab.user_destroy';
+const TOPIC_USER_ADD_GROUP = 'gitlab.user_add_to_group';
+const TOPIC_USER_REMOVE_GROUP = 'gitlab.user_remove_from_group';
+
 /**
  * Discovers users and groups from a Gitlab instance.
  * @public
@@ -67,6 +101,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   private readonly config: GitlabProviderConfig;
   private readonly integration: GitLabIntegration;
   private readonly logger: LoggerService;
+  private readonly events?: EventsService;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
   private userTransformer: UserTransformer;
@@ -77,6 +112,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: LoggerService;
+      events?: EventsService;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
       userTransformer?: UserTransformer;
@@ -96,7 +132,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       const integration = integrations.byHost(providerConfig.host);
 
       if (!providerConfig.orgEnabled) {
-        return;
+        throw new Error(`Org not enabled for ${providerConfig.id}.`);
       }
 
       if (!integration) {
@@ -137,6 +173,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     config: GitlabProviderConfig;
     integration: GitLabIntegration;
     logger: LoggerService;
+    events?: EventsService;
     taskRunner: TaskRunner;
     userTransformer?: UserTransformer;
     groupEntitiesTransformer?: GroupEntitiesTransformer;
@@ -148,6 +185,8 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       target: this.getProviderName(),
     });
     this.scheduleFn = this.createScheduleFn(options.taskRunner);
+    this.events = options.events;
+
     this.userTransformer = options.userTransformer ?? defaultUserTransformer;
     this.groupEntitiesTransformer =
       options.groupEntitiesTransformer ?? defaultGroupEntitiesTransformer;
@@ -162,6 +201,117 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
   async connect(connection: EntityProviderConnection): Promise<void> {
     this.connection = connection;
     await this.scheduleFn();
+
+    // Specifies which topics will be listened to.
+    // The topics from the original GitLab events contain only the string 'gitlab'. These are caught by the GitlabEventRouter Module, who republishes them with a more specific topic 'gitlab.<event_name>'
+    if (this.events) {
+      await this.events.subscribe({
+        id: this.getProviderName(),
+        topics: [
+          TOPIC_GROUP_CREATE,
+          TOPIC_GROUP_DESTROY,
+          TOPIC_GROUP_RENAME,
+          TOPIC_USER_CREATE,
+          TOPIC_USER_DESTROY,
+          TOPIC_USER_ADD_GROUP,
+          TOPIC_USER_REMOVE_GROUP,
+        ],
+        onEvent: async params => {
+          this.logger.info(`Received event from topic ${params.topic}`);
+
+          const addEntitiesOperation = (entities: Entity[]) => ({
+            removed: [],
+            added: entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            })),
+          });
+
+          const removeEntitiesOperation = (entities: Entity[]) => ({
+            added: [],
+            removed: entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            })),
+          });
+
+          const replaceEntitiesOperation = (entities: Entity[]) => {
+            const entitiesToReplace = entities.map(entity => ({
+              locationKey: this.getProviderName(),
+              entity: this.withLocations(
+                this.integration.config.host,
+                this.integration.config.baseUrl,
+                entity,
+              ),
+            }));
+
+            return {
+              removed: entitiesToReplace,
+              added: entitiesToReplace,
+            };
+          };
+
+          // handle group change events
+          if (
+            params.topic === TOPIC_GROUP_CREATE ||
+            params.topic === TOPIC_GROUP_DESTROY
+          ) {
+            const payload: SystemHookGroupCreateOrDestroyEventSchema =
+              params.eventPayload as SystemHookGroupCreateOrDestroyEventSchema;
+
+            const createDeltaOperation =
+              params.topic === TOPIC_GROUP_CREATE
+                ? addEntitiesOperation
+                : removeEntitiesOperation;
+
+            await this.onGroupChange(payload, createDeltaOperation);
+          }
+          if (params.topic === TOPIC_GROUP_RENAME) {
+            const payload: SystemHookGroupRenameEventSchema =
+              params.eventPayload as SystemHookGroupRenameEventSchema;
+
+            await this.onGroupEdit(payload, replaceEntitiesOperation);
+          }
+
+          // handle user change events
+          if (
+            params.topic === TOPIC_USER_CREATE ||
+            params.topic === TOPIC_USER_DESTROY
+          ) {
+            const payload: SystemHookUserCreateOrDestroyEventSchema =
+              params.eventPayload as SystemHookUserCreateOrDestroyEventSchema;
+
+            const createDeltaOperation =
+              params.topic === TOPIC_USER_CREATE
+                ? addEntitiesOperation
+                : removeEntitiesOperation;
+
+            await this.onUserChange(payload, createDeltaOperation);
+          }
+
+          // handle user membership changes
+          if (
+            params.topic === TOPIC_USER_ADD_GROUP ||
+            params.topic === TOPIC_USER_REMOVE_GROUP
+          ) {
+            const payload: SystemHookCreateOrDestroyMembershipEventsSchema =
+              params.eventPayload as SystemHookCreateOrDestroyMembershipEventsSchema;
+
+            const createDeltaOperation = addEntitiesOperation;
+
+            await this.onMembershipChange(payload, createDeltaOperation);
+          }
+        },
+      });
+    }
   }
 
   private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
@@ -208,6 +358,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       groups = paginated<GitLabGroup>(options => client.listGroups(options), {
         page: 1,
         per_page: 100,
+        all_available: true,
       });
 
       users = paginated<GitLabUser>(options => client.listUsers(options), {
@@ -229,7 +380,7 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
     const idMappedUser: { [userId: number]: GitLabUser } = {};
 
-    const res: Result = {
+    const userRes: UserResult = {
       scanned: 0,
       matches: [],
     };
@@ -240,46 +391,42 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
     };
 
     for await (const user of users) {
-      if (!this.config.userPattern.test(user.email ?? user.username ?? '')) {
-        continue;
-      }
+      userRes.scanned++;
 
-      res.scanned++;
-
-      if (user.state !== 'active') {
+      if (!this.shouldProcessUser(user)) {
+        logger.debug(`Skipped user: ${user.username}`);
         continue;
       }
 
       idMappedUser[user.id] = user;
-      res.matches.push(user);
+      userRes.matches.push(user);
     }
 
     for await (const group of groups) {
-      if (!this.config.groupPattern.test(group.full_path ?? '')) {
-        continue;
-      }
-
-      if (
-        this.config.group &&
-        !group.full_path.startsWith(`${this.config.group}/`)
-      ) {
-        continue;
-      }
-
       groupRes.scanned++;
+
+      if (!this.shouldProcessGroup(group)) {
+        logger.debug(`Skipped group: ${group.full_path}`);
+        continue;
+      }
+      logger.debug(`Processed group: ${group.full_path}`);
+
       groupRes.matches.push(group);
 
       let groupUsers: PagedResponse<GitLabUser> = { items: [] };
       try {
-        groupUsers = await client.getGroupMembers(group.full_path, ['DIRECT']);
+        const relations = this.config.allowInherited
+          ? ['DIRECT', 'INHERITED']
+          : ['DIRECT'];
+        groupUsers = await client.getGroupMembers(group.full_path, relations);
       } catch (e) {
         logger.error(
           `Failed fetching users for group '${group.full_path}': ${e}`,
         );
       }
-
       for (const groupUser of groupUsers.items) {
         const user = idMappedUser[groupUser.id];
+
         if (user) {
           user.groups = (user.groups ?? []).concat(group);
         }
@@ -288,13 +435,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
 
     const groupsWithUsers = groupRes.matches.filter(group => {
       return (
-        res.matches.filter(x => {
+        userRes.matches.filter(x => {
           return !!x.groups?.find(y => y.id === group.id);
         }).length > 0
       );
     });
 
-    const userEntities = res.matches.map(p =>
+    const userEntities = userRes.matches.map(p =>
       this.userTransformer({
         user: p,
         integrationConfig: this.integration.config,
@@ -309,6 +456,13 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       groupNameTransformer: this.groupNameTransformer,
     });
 
+    logger.info(
+      `Scanned ${userRes.scanned} users and processed ${userEntities.length} users`,
+    );
+    logger.info(
+      `Scanned ${groupRes.scanned} groups and processed ${groupEntities.length} groups`,
+    );
+
     await this.connection.applyMutation({
       type: 'full',
       entities: [...userEntities, ...groupEntities].map(entity => ({
@@ -321,6 +475,301 @@ export class GitlabOrgDiscoveryEntityProvider implements EntityProvider {
       })),
     });
   }
+  private async onGroupChange(
+    event: SystemHookGroupCreateOrDestroyEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    const getClient = () =>
+      new GitLabClient({
+        config: this.integration.config,
+        logger: this.logger,
+      });
+
+    let group: GitLabGroup | undefined;
+    if (event.event_name === 'group_destroy') {
+      group = {
+        id: event.group_id,
+        full_path: event.full_path,
+        name: event.name,
+        description: '',
+        parent_id: 0,
+      };
+    } else {
+      const client = getClient();
+      group = await client.getGroupById(event.group_id);
+    }
+
+    if (!this.shouldProcessGroup(group)) {
+      this.logger.debug(`Skipped group ${group.full_path}.`);
+      return;
+    }
+
+    // create the group entity
+    const groupEntity = this.groupEntitiesTransformer({
+      groups: [group],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    // we need to fetch the parent group's object because its representation might be changed by the groupTransformer
+    if (group.parent_id) {
+      const client = new GitLabClient({
+        config: this.integration.config,
+        logger: this.logger,
+      });
+
+      const parentGroup = await client.getGroupById(group.parent_id);
+
+      groupEntity[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    this.logger.debug(`Applying mutation for group ${group.full_path}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      ...createDeltaOperation(groupEntity),
+    });
+  }
+
+  // the goal here is to trigger a mutation to remove the old entity and add the new one.
+  private async onGroupEdit(
+    event: SystemHookGroupRenameEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    const groupToRemove: GitLabGroup = {
+      id: event.group_id,
+      full_path: event.old_full_path,
+      name: event.name,
+      description: '',
+      parent_id: 0,
+    };
+
+    if (!this.shouldProcessGroup(groupToRemove)) {
+      this.logger.debug(`Skipped group ${groupToRemove.full_path}.`);
+      return;
+    }
+
+    const groupEntityToRemove = await this.groupEntitiesTransformer({
+      groups: [groupToRemove],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    const client = new GitLabClient({
+      config: this.integration.config,
+      logger: this.logger,
+    });
+
+    const groupToAdd = await client.getGroupById(event.group_id);
+
+    if (!this.shouldProcessGroup(groupToAdd)) {
+      this.logger.debug(`Skipped group ${groupToAdd.full_path}.`);
+      return;
+    }
+
+    const groupEntityToAdd = await this.groupEntitiesTransformer({
+      groups: [groupToAdd],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    if (groupToAdd.parent_id) {
+      const parentGroup = await client.getGroupById(groupToAdd.parent_id);
+
+      groupEntityToAdd[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    const { added } = createDeltaOperation([...groupEntityToAdd]);
+    const { removed } = createDeltaOperation([...groupEntityToRemove]);
+
+    this.logger.debug(`Applying mutation for group ${groupToAdd.full_path}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onUserChange(
+    event: SystemHookUserCreateOrDestroyEventSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    let user: GitLabUser | undefined;
+    if (event.event_name === 'user_destroy') {
+      user = {
+        id: event.user_id,
+        username: event.username,
+        email: event.email,
+        name: event.name,
+        state: 'active', // in the delete case it doesn't really matter if the user is active or not
+        web_url: '',
+        avatar_url: '',
+      };
+    } else {
+      const client = new GitLabClient({
+        config: this.integration.config,
+        logger: this.logger,
+      });
+
+      user = await client.getUserById(event.user_id);
+    }
+
+    if (!this.shouldProcessUser(user)) {
+      this.logger.debug(`Skipped user ${user.username}.`);
+      return;
+    }
+
+    const userEntity = await this.userTransformer({
+      user: user,
+      integrationConfig: this.integration.config,
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+    const { added, removed } = createDeltaOperation([userEntity]);
+
+    this.logger.debug(`Applying mutation for user ${user.username}.`);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  // the goal here is to reconstruct the group either from which the user was removed or to which the user was added. Specifically, we add/remove the new user to/from the spec.member property array of the group entity. The Processor should take care of updating the relations
+  private async onMembershipChange(
+    event: SystemHookCreateOrDestroyMembershipEventsSchema,
+    createDeltaOperation: (entities: Entity[]) => {
+      added: any[];
+      removed: any[];
+    },
+  ): Promise<void> {
+    if (!this.connection) {
+      throw new Error(
+        `Gitlab discovery connection not initialized for ${this.getProviderName()}`,
+      );
+    }
+
+    const client = new GitLabClient({
+      config: this.integration.config,
+      logger: this.logger,
+    });
+
+    const groupToRebuild: GitLabGroup = await client.getGroupById(
+      event.group_id,
+    );
+
+    // If the group is outside the scope there is no point creating anything related to it.
+    if (!this.shouldProcessGroup(groupToRebuild)) {
+      this.logger.debug(`Skipped group ${groupToRebuild.full_path}.`);
+      return;
+    }
+
+    const relations = this.config.allowInherited
+      ? ['DIRECT', 'INHERITED']
+      : ['DIRECT'];
+
+    const groupMembers = await client.getGroupMembers(
+      groupToRebuild.full_path,
+      relations,
+    );
+
+    // if (groupMembers.items.length !== 0) {
+    //   groupMembers.items.forEach(element => {
+    //     if (
+    //       event.event_name === 'user_remove_from_group' &&
+    //       element.username === event.user_username
+    //     ) {
+    //       return;
+    //     }
+
+    //     usersToBeAdded.push(element.username);
+    //   });
+    // }
+
+    // new members of the group
+    const usersToBeAdded =
+      event.event_name === 'user_remove_from_group'
+        ? groupMembers.items.filter(e => e.username !== event.user_username)
+        : groupMembers.items;
+
+    const groupEntityToModify = await this.groupEntitiesTransformer({
+      groups: [groupToRebuild],
+      providerConfig: this.config,
+      groupNameTransformer: this.groupNameTransformer,
+    });
+
+    // we need to fetch the parent group's object because its representation might be changed by the groupTransformer
+    if (groupToRebuild.parent_id) {
+      const parentGroup = await client.getGroupById(groupToRebuild.parent_id);
+
+      groupEntityToModify[0].spec.parent = this.groupNameTransformer({
+        group: parentGroup,
+        providerConfig: this.config,
+      });
+    }
+
+    if (usersToBeAdded.length !== 0) {
+      groupEntityToModify[0].spec.members = usersToBeAdded.map(e => e.username);
+    }
+
+    const { added, removed } = createDeltaOperation([...groupEntityToModify]);
+
+    this.logger.debug(
+      `Applying mutation for group ${groupToRebuild.full_path}.`,
+    );
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private shouldProcessGroup(group: GitLabGroup): boolean {
+    return (
+      this.config.groupPattern.test(group.full_path) &&
+      (!this.config.group ||
+        group.full_path.startsWith(`${this.config.group}/`))
+    );
+  }
+
+  private shouldProcessUser(user: GitLabUser): boolean {
+    return (
+      this.config.userPattern.test(user.email ?? user.username ?? '') &&
+      user.state === 'active'
+    );
+  }
 
   private withLocations(host: string, baseUrl: string, entity: Entity): Entity {
     const location =

yarn.lock

@@ -5569,7 +5569,24 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab":
+"@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org":
+  version: 0.0.0-use.local
+  resolution: "@backstage/plugin-catalog-backend-module-gitlab-org@workspace:plugins/catalog-backend-module-gitlab-org"
+  dependencies:
+    "@backstage/backend-common": "workspace:^"
+    "@backstage/backend-plugin-api": "workspace:^"
+    "@backstage/backend-tasks": "workspace:^"
+    "@backstage/backend-test-utils": "workspace:^"
+    "@backstage/cli": "workspace:^"
+    "@backstage/plugin-catalog-backend-module-gitlab": "workspace:^"
+    "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-events-backend-test-utils": "workspace:^"
+    "@backstage/plugin-events-node": "workspace:^"
+    luxon: ^3.0.0
+  languageName: unknown
+  linkType: soft
+
+"@backstage/plugin-catalog-backend-module-gitlab@workspace:^, @backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab":
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-backend-module-gitlab@workspace:plugins/catalog-backend-module-gitlab"
   dependencies:
@@ -5581,7 +5598,11 @@ __metadata:
     "@backstage/cli": "workspace:^"
     "@backstage/config": "workspace:^"
     "@backstage/integration": "workspace:^"
+    "@backstage/plugin-catalog-common": "workspace:^"
     "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-events-backend-test-utils": "workspace:^"
+    "@backstage/plugin-events-node": "workspace:^"
+    "@gitbeaker/rest": ^39.25.0
     "@types/lodash": ^4.14.151
     "@types/uuid": ^9.0.0
     lodash: ^4.17.21