introduce a catalog aws module

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

plugins/catalog-backend-module-aws/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/catalog-backend-module-aws/README.md

@@ -0,0 +1,10 @@
+# Catalog Backend Module for LDAP
+
+This is an extension module to the plugin-catalog-backend plugin, providing an
+`AwsOrganizationCloudAccountProcessor` that can be used to ingest cloud accounts
+as `Resource` kind entities.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/integrations/ldap/org) for details on how to install
+and configure the plugin.

plugins/catalog-backend-module-aws/api-report.md

@@ -0,0 +1,30 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-aws"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { LocationSpec } from '@backstage/catalog-model';
+import { Logger as Logger_2 } from 'winston';
+
+// @public
+export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger_2;
+    },
+  ): AwsOrganizationCloudAccountProcessor;
+  // (undocumented)
+  getProcessorName(): string;
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean>;
+}
+```

plugins/catalog-backend-module-aws/config.d.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  catalog?: {
+    /**
+     * List of processor-specific options and attributes
+     */
+    processors?: {
+      /**
+       * AwsOrganizationCloudAccountProcessor configuration
+       */
+      awsOrganization?: {
+        provider: {
+          /**
+           * The role to be assumed by this processor
+           */
+          roleArn?: string;
+        };
+      };
+    };
+  };
+}

plugins/catalog-backend-module-aws/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-aws",
+  "description": "A Backstage catalog backend module that helps integrate towards AWS",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-aws"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean",
+    "start": "backstage-cli package start"
+  },
+  "dependencies": {
+    "@backstage/catalog-model": "^0.10.0",
+    "@backstage/config": "^0.1.14",
+    "@backstage/errors": "^0.2.1",
+    "@backstage/plugin-catalog-backend": "^0.21.4",
+    "@backstage/types": "^0.1.2",
+    "aws-sdk": "^2.840.0",
+    "lodash": "^4.17.21",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.14.0",
+    "@types/lodash": "^4.14.151",
+    "aws-sdk-mock": "^5.2.1"
+  },
+  "files": [
+    "dist",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
+}

plugins/catalog-backend-module-aws/src/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A Backstage catalog backend module that helps integrate towards AWS
+ *
+ * @packageDocumentation
+ */
+
+export * from './processors';

plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts

@@ -15,13 +15,11 @@
  */
 
 import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
-import * as winston from 'winston';
 
 describe('AwsOrganizationCloudAccountProcessor', () => {
   describe('readLocation', () => {
-    const processor = new AwsOrganizationCloudAccountProcessor({
+    const processor = new (AwsOrganizationCloudAccountProcessor as any)({
       provider: {},
-      logger: winston.createLogger(),
     });
     const location = { type: 'aws-cloud-accounts', target: '' };
     const emit = jest.fn();

plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts

@@ -13,35 +13,40 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import { LocationSpec, ResourceEntityV1alpha1 } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  results,
+} from '@backstage/plugin-catalog-backend';
 import AWS, { Credentials, Organizations } from 'aws-sdk';
 import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';
-
-import * as results from './results';
-import { CatalogProcessor, CatalogProcessorEmit } from './types';
-import { Config } from '@backstage/config';
 import { Logger } from 'winston';
 import {
   AwsOrganizationProviderConfig,
   readAwsOrganizationConfig,
-} from './awsOrganization/config';
+} from '../awsOrganization/config';
 
 const AWS_ORGANIZATION_REGION = 'us-east-1';
 const LOCATION_TYPE = 'aws-cloud-accounts';
 
-const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id';
-const ARN_ANNOTATION: string = 'amazonaws.com/arn';
-const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id';
+const ACCOUNTID_ANNOTATION = 'amazonaws.com/account-id';
+const ARN_ANNOTATION = 'amazonaws.com/arn';
+const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';
 
 /**
  * A processor for ingesting AWS Accounts from AWS Organizations.
  *
- * If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object.
+ * If custom authentication is needed, it can be achieved by configuring the
+ * global AWS.credentials object.
+ *
+ * @public
  */
 export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
-  logger: Logger;
-  organizations: Organizations;
-  provider: AwsOrganizationProviderConfig;
+  private readonly organizations: Organizations;
+  private readonly provider: AwsOrganizationProviderConfig;
 
   static fromConfig(config: Config, options: { logger: Logger }) {
     const c = config.getOptionalConfig('catalog.processors.awsOrganization');
@@ -67,12 +72,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
     });
   }
 
-  constructor(options: {
-    provider: AwsOrganizationProviderConfig;
-    logger: Logger;
-  }) {
+  private constructor(options: { provider: AwsOrganizationProviderConfig }) {
     this.provider = options.provider;
-    this.logger = options.logger;
     const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(
       this.provider,
     );
@@ -81,70 +82,11 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
       region: AWS_ORGANIZATION_REGION,
     }); // Only available in us-east-1
   }
+
   getProcessorName(): string {
     return 'AwsOrganizationCloudAccountProcessor';
   }
 
-  normalizeName(name: string): string {
-    return name
-      .trim()
-      .toLocaleLowerCase()
-      .replace(/[^a-zA-Z0-9\-]/g, '-');
-  }
-
-  extractInformationFromArn(arn: string): {
-    accountId: string;
-    organizationId: string;
-  } {
-    const parts = arn.split('/');
-
-    return {
-      accountId: parts[parts.length - 1],
-      organizationId: parts[parts.length - 2],
-    };
-  }
-
-  async getAwsAccounts(): Promise<Account[]> {
-    let awsAccounts: Account[] = [];
-    let isInitialAttempt = true;
-    let nextToken = undefined;
-    while (isInitialAttempt || nextToken) {
-      isInitialAttempt = false;
-      const orgAccounts: ListAccountsResponse = await this.organizations
-        .listAccounts({ NextToken: nextToken })
-        .promise();
-      if (orgAccounts.Accounts) {
-        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
-      }
-      nextToken = orgAccounts.NextToken;
-    }
-
-    return awsAccounts;
-  }
-
-  mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {
-    const { accountId, organizationId } = this.extractInformationFromArn(
-      account.Arn as string,
-    );
-    return {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Resource',
-      metadata: {
-        annotations: {
-          [ACCOUNTID_ANNOTATION]: accountId,
-          [ARN_ANNOTATION]: account.Arn || '',
-          [ORGANIZATION_ANNOTATION]: organizationId,
-        },
-        name: this.normalizeName(account.Name || ''),
-        namespace: 'default',
-      },
-      spec: {
-        type: 'cloud-account',
-        owner: 'unknown',
-      },
-    };
-  }
-
   async readLocation(
     location: LocationSpec,
     _optional: boolean,
@@ -168,10 +110,70 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
         }
         return true;
       })
-      .forEach((entity: ResourceEntityV1alpha1) => {
+      .forEach(entity => {
         emit(results.entity(location, entity));
       });
 
     return true;
   }
+
+  private normalizeName(name: string): string {
+    return name
+      .trim()
+      .toLocaleLowerCase('en-US')
+      .replace(/[^a-zA-Z0-9\-]/g, '-');
+  }
+
+  private extractInformationFromArn(arn: string): {
+    accountId: string;
+    organizationId: string;
+  } {
+    const parts = arn.split('/');
+
+    return {
+      accountId: parts[parts.length - 1],
+      organizationId: parts[parts.length - 2],
+    };
+  }
+
+  private async getAwsAccounts(): Promise<Account[]> {
+    let awsAccounts: Account[] = [];
+    let isInitialAttempt = true;
+    let nextToken = undefined;
+    while (isInitialAttempt || nextToken) {
+      isInitialAttempt = false;
+      const orgAccounts: ListAccountsResponse = await this.organizations
+        .listAccounts({ NextToken: nextToken })
+        .promise();
+      if (orgAccounts.Accounts) {
+        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
+      }
+      nextToken = orgAccounts.NextToken;
+    }
+
+    return awsAccounts;
+  }
+
+  private mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {
+    const { accountId, organizationId } = this.extractInformationFromArn(
+      account.Arn as string,
+    );
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Resource',
+      metadata: {
+        annotations: {
+          [ACCOUNTID_ANNOTATION]: accountId,
+          [ARN_ANNOTATION]: account.Arn || '',
+          [ORGANIZATION_ANNOTATION]: organizationId,
+        },
+        name: this.normalizeName(account.Name || ''),
+        namespace: 'default',
+      },
+      spec: {
+        type: 'cloud-account',
+        owner: 'unknown',
+      },
+    };
+  }
 }

plugins/catalog-backend-module-aws/src/processors/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';

plugins/catalog-backend-module-aws/src/setupTests.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};

plugins/catalog-backend-module-ldap/config.d.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { JsonValue } from '@backstage/config';
+import { JsonValue } from '@backstage/types';
 
 export interface Config {
   /**

plugins/catalog-backend-module-ldap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-ldap",
-  "description": "A Backstage catalog backend modules that helps integrate towards LDAP",
+  "description": "A Backstage catalog backend module that helps integrate towards LDAP",
   "version": "0.3.13",
   "main": "src/index.ts",
   "types": "src/index.ts",

plugins/catalog-backend-module-ldap/src/index.ts

@@ -15,7 +15,7 @@
  */
 
 /**
- * A Backstage catalog backend modules that helps integrate towards LDAP
+ * A Backstage catalog backend module that helps integrate towards LDAP
  *
  * @packageDocumentation
  */

plugins/catalog-backend-module-msgraph/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-msgraph",
-  "description": "A Backstage catalog backend modules that helps integrate towards Microsoft Graph",
+  "description": "A Backstage catalog backend module that helps integrate towards Microsoft Graph",
   "version": "0.2.16",
   "main": "src/index.ts",
   "types": "src/index.ts",

plugins/catalog-backend-module-msgraph/src/index.ts

@@ -15,7 +15,7 @@
  */
 
 /**
- * A Backstage catalog backend modules that helps integrate towards Microsoft Graph
+ * A Backstage catalog backend module that helps integrate towards Microsoft Graph
  *
  * @packageDocumentation
  */

plugins/catalog-backend/api-report.md

@@ -5,7 +5,6 @@
 ```ts
 /// <reference types="node" />
 
-import { Account } from 'aws-sdk/clients/organizations';
 import { BitbucketIntegration } from '@backstage/integration';
 import { CatalogApi } from '@backstage/catalog-client';
 import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node';
@@ -25,7 +24,6 @@ import { JsonValue } from '@backstage/types';
 import { Location as Location_2 } from '@backstage/catalog-client';
 import { LocationSpec } from '@backstage/catalog-model';
 import { Logger as Logger_2 } from 'winston';
-import { Organizations } from 'aws-sdk';
 import { Permission } from '@backstage/plugin-permission-common';
 import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
 import { PermissionCondition } from '@backstage/plugin-permission-common';
@@ -33,7 +31,6 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common';
 import { PermissionRule } from '@backstage/plugin-permission-node';
 import { PluginDatabaseManager } from '@backstage/backend-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
-import { ResourceEntityV1alpha1 } from '@backstage/catalog-model';
 import { Router } from 'express';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 import { TokenManager } from '@backstage/backend-common';
@@ -115,55 +112,6 @@ export class AnnotateScmSlugEntityProcessor implements CatalogProcessor {
   preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
 }
 
-// Warning: (ae-missing-release-tag) "AwsOrganizationCloudAccountProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
-export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
-  constructor(options: {
-    provider: AwsOrganizationProviderConfig;
-    logger: Logger_2;
-  });
-  // (undocumented)
-  extractInformationFromArn(arn: string): {
-    accountId: string;
-    organizationId: string;
-  };
-  // (undocumented)
-  static fromConfig(
-    config: Config,
-    options: {
-      logger: Logger_2;
-    },
-  ): AwsOrganizationCloudAccountProcessor;
-  // (undocumented)
-  getAwsAccounts(): Promise<Account[]>;
-  // (undocumented)
-  getProcessorName(): string;
-  // (undocumented)
-  logger: Logger_2;
-  // (undocumented)
-  mapAccountToComponent(account: Account): ResourceEntityV1alpha1;
-  // (undocumented)
-  normalizeName(name: string): string;
-  // (undocumented)
-  organizations: Organizations;
-  // (undocumented)
-  provider: AwsOrganizationProviderConfig;
-  // (undocumented)
-  readLocation(
-    location: LocationSpec,
-    _optional: boolean,
-    emit: CatalogProcessorEmit,
-  ): Promise<boolean>;
-}
-
-// Warning: (ae-missing-release-tag) "AwsOrganizationProviderConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
-export type AwsOrganizationProviderConfig = {
-  roleArn?: string;
-};
-
 // Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/catalog-backend/config.d.ts

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-import { JsonValue } from '@backstage/config';
-
 export interface Config {
   /**
    * Configuration options for the catalog plugin.
@@ -172,19 +170,6 @@ export interface Config {
           userNamespace?: string;
         }>;
       };
-
-      /**
-       * AwsOrganizationCloudAccountProcessor configuration
-       */
-      awsOrganization?: {
-        provider: {
-          /**
-           * The role to be assumed by this processor
-           *
-           */
-          roleArn?: string;
-        };
-      };
     };
   };
 }

plugins/catalog-backend/package.json

@@ -47,7 +47,6 @@
     "@backstage/types": "^0.1.2",
     "@octokit/graphql": "^4.5.8",
     "@types/express": "^4.17.6",
-    "aws-sdk": "^2.840.0",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",
     "express": "^4.17.1",
@@ -79,7 +78,6 @@
     "@types/supertest": "^2.0.8",
     "@types/uuid": "^8.0.0",
     "@vscode/sqlite3": "^5.0.7",
-    "aws-sdk-mock": "^5.2.1",
     "msw": "^0.35.0",
     "supertest": "^6.1.3",
     "wait-for-expect": "^3.0.2",

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -18,8 +18,6 @@ import * as results from './results';
 
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
-export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
-export type { AwsOrganizationProviderConfig } from './awsOrganization/config';
 export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';