diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index 91e15dda5f..487eba823f 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -4,6 +4,7 @@ ```ts import { AwsCredentialIdentity } from '@aws-sdk/types'; +import { AwsCredentialsManager } from '@backstage/integration-aws-node'; import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { CatalogProcessorParser } from '@backstage/plugin-catalog-node'; @@ -23,7 +24,12 @@ export type AWSCredentialFactory = ( // @public export class AwsEKSClusterProcessor implements CatalogProcessor { - constructor(options: { credentialsFactory?: AWSCredentialFactory }); + constructor(options: { + credentialsFactory?: AWSCredentialFactory; + credentialsManager?: AwsCredentialsManager; + }); + // (undocumented) + static fromConfig(configRoot: Config): AwsEKSClusterProcessor; // (undocumented) getProcessorName(): string; // (undocumented) @@ -42,9 +48,9 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { static fromConfig( config: Config, options: { - logger: Logger; + logger?: Logger; }, - ): AwsOrganizationCloudAccountProcessor; + ): Promise; // (undocumented) getProcessorName(): string; // (undocumented) diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 2774802d25..d5fbaafdd2 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -58,6 +58,7 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/integration-aws-node": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-kubernetes-common": "workspace:^", @@ -73,6 +74,7 @@ "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "aws-sdk-client-mock": "^2.0.0", + "aws-sdk-client-mock-jest": "^2.0.0", "luxon": "^3.0.0", "yaml": "^2.0.0" }, diff --git a/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts b/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts deleted file mode 100644 index b707f0bc4e..0000000000 --- a/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - AwsCredentialIdentity, - AwsCredentialIdentityProvider, -} from '@aws-sdk/types'; -import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; - -export class AwsCredentials { - /** - * If accessKeyId and secretAccessKey are missing, the DefaultAWSCredentialsProviderChain will be used: - * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html - */ - static create( - config: { - accessKeyId?: string; - secretAccessKey?: string; - roleArn?: string; - externalId?: string; - }, - roleSessionName: string, - ): AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined { - if (!config) { - return undefined; - } - - const accessKeyId = config.accessKeyId; - const secretAccessKey = config.secretAccessKey; - let explicitCredentials: AwsCredentialIdentity | undefined; - - if (accessKeyId && secretAccessKey) { - explicitCredentials = { - accessKeyId, - secretAccessKey, - }; - } - - const roleArn = config.roleArn; - if (roleArn) { - return fromTemporaryCredentials({ - masterCredentials: explicitCredentials, - params: { - RoleArn: roleArn, - RoleSessionName: roleSessionName, - ExternalId: config.externalId, - }, - }); - } - - return explicitCredentials; - } -} diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts index cf66d72fb5..4a19b9eb7f 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsEKSClusterProcessor.ts @@ -24,9 +24,14 @@ import { ANNOTATION_KUBERNETES_API_SERVER_CA, ANNOTATION_KUBERNETES_AUTH_PROVIDER, } from '@backstage/plugin-kubernetes-common'; -import { AwsCredentialIdentity } from '@aws-sdk/types'; import { EKS } from '@aws-sdk/client-eks'; import { AWSCredentialFactory } from '../types'; +import { AwsCredentialIdentity, Provider } from '@aws-sdk/types'; +import { + AwsCredentialsManager, + DefaultAwsCredentialsManager, +} from '@backstage/integration-aws-node'; +import { Config } from '@backstage/config'; const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id'; const ARN_ANNOTATION: string = 'amazonaws.com/arn'; @@ -40,9 +45,22 @@ const ARN_ANNOTATION: string = 'amazonaws.com/arn'; */ export class AwsEKSClusterProcessor implements CatalogProcessor { private credentialsFactory?: AWSCredentialFactory; + private credentialsManager?: AwsCredentialsManager; - constructor(options: { credentialsFactory?: AWSCredentialFactory }) { + static fromConfig(configRoot: Config): AwsEKSClusterProcessor { + const awsCredentaislManager = + DefaultAwsCredentialsManager.fromConfig(configRoot); + return new AwsEKSClusterProcessor({ + credentialsManager: awsCredentaislManager, + }); + } + + constructor(options: { + credentialsFactory?: AWSCredentialFactory; + credentialsManager?: AwsCredentialsManager; + }) { this.credentialsFactory = options.credentialsFactory; + this.credentialsManager = options.credentialsManager; } getProcessorName(): string { @@ -80,7 +98,17 @@ export class AwsEKSClusterProcessor implements CatalogProcessor { credentials = await this.credentialsFactory(accountId); } - const eksClient = new EKS({ credentials, region }); + let providerFunction: (() => Provider) | undefined; + if (this.credentialsManager) { + const credentialsProvider = + await this.credentialsManager.getCredentialProvider({ accountId }); + providerFunction = () => credentialsProvider.sdkCredentialProvider; + } + + const eksClient = new EKS({ + credentials, + credentialDefaultProvider: providerFunction, + }); const clusters = await eksClient.listClusters({}); if (clusters.clusters === undefined) { return true; diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts index 40637ba99e..8ebfaec441 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -15,6 +15,7 @@ */ import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; import { ListAccountsCommand, @@ -32,7 +33,6 @@ describe('AwsOrganizationCloudAccountProcessor', () => { afterEach(() => { jest.resetAllMocks(); - mock.reset(); }); it('generates component entities for accounts', async () => { diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts index 8ef393f2b0..b4fc2a8272 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts @@ -27,13 +27,12 @@ import { ListAccountsResponse, Organizations, } from '@aws-sdk/client-organizations'; -import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; -import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { Logger } from 'winston'; +import { readAwsOrganizationConfig } from '../awsOrganization/config'; import { - AwsOrganizationProviderConfig, - readAwsOrganizationConfig, -} from '../awsOrganization/config'; + AwsCredentialProvider, + DefaultAwsCredentialsManager, +} from '@backstage/integration-aws-node'; const AWS_ORGANIZATION_REGION = 'us-east-1'; const LOCATION_TYPE = 'aws-cloud-accounts'; @@ -52,39 +51,31 @@ const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id'; */ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { private readonly organizations: Organizations; - private readonly provider: AwsOrganizationProviderConfig; + private readonly logger?: Logger; - static fromConfig(config: Config, options: { logger: Logger }) { + static async fromConfig(config: Config, options: { logger?: Logger }) { const c = config.getOptionalConfig('catalog.processors.awsOrganization'); - return new AwsOrganizationCloudAccountProcessor({ - ...options, - provider: c ? readAwsOrganizationConfig(c) : {}, + const orgConfig = c ? readAwsOrganizationConfig(c) : undefined; + const awsCredentialsManager = + DefaultAwsCredentialsManager.fromConfig(config); + const credProvider = await awsCredentialsManager.getCredentialProvider({ + arn: orgConfig?.roleArn, }); - } - - private static buildCredentials( - config: AwsOrganizationProviderConfig, - ): AwsCredentialIdentityProvider | undefined { - const roleArn = config.roleArn; - if (!roleArn) { - return undefined; - } - - return fromTemporaryCredentials({ - params: { - RoleSessionName: 'backstage-aws-organization-processor', - RoleArn: roleArn, - }, - }); - } - - private constructor(options: { provider: AwsOrganizationProviderConfig }) { - this.provider = options.provider; - const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials( - this.provider, + return new AwsOrganizationCloudAccountProcessor( + credProvider, + options.logger, ); + } + + private constructor( + private readonly credProvider: AwsCredentialProvider, + logger?: Logger, + ) { + this.logger = logger?.child({ + target: this.getProcessorName(), + }); this.organizations = new Organizations({ - credentials, + credentialDefaultProvider: () => this.credProvider.sdkCredentialProvider, region: AWS_ORGANIZATION_REGION, }); // Only available in us-east-1 } @@ -102,6 +93,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { return false; } + this.logger?.info('Discovering AWS Organization Account objects'); + (await this.getAwsAccounts()) .map(account => this.mapAccountToComponent(account)) .filter(entity => { diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts index 70a03ad6ba..8354a34e68 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts @@ -24,6 +24,7 @@ import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { AwsS3EntityProvider } from './AwsS3EntityProvider'; import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; import { ListObjectsV2Command, S3 } from '@aws-sdk/client-s3'; class PersistingTaskRunner implements TaskRunner { @@ -55,8 +56,6 @@ describe('AwsS3EntityProvider', () => { const mock = mockClient(S3); beforeEach(() => { - mock.reset(); - mock.on(ListObjectsV2Command).callsFake(async req => { const prefix = req.Prefix ?? ''; diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts index 41b3a4cbcc..4a87a42e75 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts @@ -23,7 +23,6 @@ import { locationSpecToLocationEntity, } from '@backstage/plugin-catalog-node'; import { LocationSpec } from '@backstage/plugin-catalog-common'; -import { AwsCredentials } from '../credentials/AwsCredentials'; import { readAwsS3Configs } from './config'; import { AwsS3Config } from './types'; import { @@ -34,6 +33,10 @@ import { import * as uuid from 'uuid'; import { Logger } from 'winston'; import { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint'; +import { + AwsCredentialsManager, + DefaultAwsCredentialsManager, +} from '@backstage/integration-aws-node'; // TODO: event-based updates using S3 events (+ queue like SQS)? /** @@ -45,7 +48,7 @@ import { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint'; */ export class AwsS3EntityProvider implements EntityProvider { private readonly logger: Logger; - private readonly s3: S3; + private s3?: S3; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; private endpoint?: string; @@ -83,7 +86,8 @@ export class AwsS3EntityProvider implements EntityProvider { `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`, ); } - + const awsCredentialsManager = + DefaultAwsCredentialsManager.fromConfig(configRoot); const taskRunner = options.schedule ?? options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); @@ -91,6 +95,7 @@ export class AwsS3EntityProvider implements EntityProvider { return new AwsS3EntityProvider( providerConfig, integration, + awsCredentialsManager, options.logger, taskRunner, ); @@ -99,7 +104,8 @@ export class AwsS3EntityProvider implements EntityProvider { private constructor( private readonly config: AwsS3Config, - integration: AwsS3Integration, + private readonly integration: AwsS3Integration, + private readonly awsCredentialsManager: AwsCredentialsManager, logger: Logger, taskRunner: TaskRunner, ) { @@ -107,16 +113,6 @@ export class AwsS3EntityProvider implements EntityProvider { target: this.getProviderName(), }); - this.s3 = new S3({ - apiVersion: '2006-03-01', - credentials: AwsCredentials.create( - integration.config, - 'backstage-aws-s3-provider', - ), - endpoint: integration.config.endpoint, - region: this.config.region, - forcePathStyle: integration.config.s3ForcePathStyle, - }); this.scheduleFn = this.createScheduleFn(taskRunner); } @@ -150,6 +146,16 @@ export class AwsS3EntityProvider implements EntityProvider { /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection): Promise { this.connection = connection; + const credProvider = + await this.awsCredentialsManager.getCredentialProvider(); + this.s3 = new S3({ + apiVersion: '2006-03-01', + credentialDefaultProvider: () => credProvider.sdkCredentialProvider, + endpoint: this.integration.config.endpoint, + region: this.config.region, + forcePathStyle: this.integration.config.s3ForcePathStyle, + }); + // https://github.com/aws/aws-sdk-js-v3/issues/4122#issuecomment-1298968804 const endpoint = await getEndpointFromInstructions( { @@ -191,6 +197,10 @@ export class AwsS3EntityProvider implements EntityProvider { } private async listAllObjectKeys(): Promise { + if (!this.s3) { + throw new Error('Not initialized'); + } + const keys: string[] = []; let continuationToken: string | undefined = undefined; diff --git a/yarn.lock b/yarn.lock index e7280a759f..aa759e1dbd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5840,12 +5840,14 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/integration-aws-node": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-kubernetes-common": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 aws-sdk-client-mock: ^2.0.0 + aws-sdk-client-mock-jest: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 p-limit: ^3.0.2