diff --git a/.changeset/silly-knives-warn.md b/.changeset/silly-knives-warn.md new file mode 100644 index 0000000000..f5fb61b493 --- /dev/null +++ b/.changeset/silly-knives-warn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-node': patch +--- + +Add support for specifying an S3 bucket's account ID and retrieving the credentials from the `aws` app config section. This is now the preferred way to configure AWS credentials for Techdocs. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4e806296e5..097d31f760 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -106,8 +106,20 @@ techdocs: # If not set, the default location will be the root of the storage bucket bucketRootPath: '/' - # (Optional) An API key is required to write to a storage bucket. - # If not set, environment variables or aws config file will be used to authenticate. + # (Optional) The AWS account ID where the storage bucket is located. + # Credentials for the account ID must be configured in the 'aws' app config section. + # See the integration-aws-node package for details on how to configure credentials in + # the 'aws' app config section. + # https://www.npmjs.com/package/@backstage/integration-aws-node + # If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate. + # https://www.npmjs.com/package/@aws-sdk/credential-provider-node + # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html + accountId: ${TECHDOCS_AWSS3_ACCOUNT_ID} + + # (Optional) AWS credentials to use to write to the storage bucket. + # This configuration section is now deprecated. + # Configuring the account ID is now preferred, with credentials in the 'aws' app config section. + # If credentials are not set and no account ID is set, environment variables or aws config file will be used to authenticate. # https://www.npmjs.com/package/@aws-sdk/credential-provider-node # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html credentials: diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 3ea57edc6b..f38a806413 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -244,10 +244,13 @@ techdocs: type: 'awsS3' awsS3: bucketName: 'name-of-techdocs-storage-bucket' + accountId: '123456789012' region: ${AWS_REGION} - credentials: - accessKeyId: ${AWS_ACCESS_KEY_ID} - secretAccessKey: ${AWS_SECRET_ACCESS_KEY} +aws: + accounts: + - accountId: '123456789012' + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} ``` Refer to the diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index ec35505a78..3a05beabdc 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -81,9 +81,23 @@ export interface Config { * Required when 'type' is set to awsS3 */ awsS3?: { + /** + * (Optional) The AWS account ID where the storage bucket is located. + * Credentials for the account ID will be sourced from the 'aws' app config section. + * See the + * [integration-aws-node package](https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md) + * for details on how to configure the credentials in the app config. + * If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate. + * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html + * @visibility secret + */ + accountId?: string; /** * (Optional) Credentials used to access a storage bucket. - * If not set, environment variables or aws config file will be used to authenticate. + * This section is now deprecated. Configuring the account ID is now preferred, with credentials in the 'aws' + * app config section. + * If not set and no account ID is set, environment variables or aws config file will be used to authenticate. * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html * @visibility secret diff --git a/plugins/techdocs-node/package.json b/plugins/techdocs-node/package.json index 668fe58239..631de02b55 100644 --- a/plugins/techdocs-node/package.json +++ b/plugins/techdocs-node/package.json @@ -50,6 +50,7 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/integration-aws-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@google-cloud/storage": "^6.0.0", "@trendyol-js/openstack-swift-sdk": "^0.0.5", diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts index 08facf2208..ff61ac58a2 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts @@ -27,6 +27,11 @@ import { import { getVoidLogger } from '@backstage/backend-common'; import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import { + AwsCredentials, + AwsCredentialsProviderOptions, + DefaultAwsCredentialsProvider, +} from '@backstage/integration-aws-node'; import { mockClient, AwsClientStub } from 'aws-sdk-client-mock'; import express from 'express'; import request from 'supertest'; @@ -40,6 +45,21 @@ import { Readable } from 'stream'; const env = process.env; let s3Mock: AwsClientStub; +function getMockCredentials(): Promise { + return Promise.resolve({ + provider: async () => { + return Promise.resolve({ + accessKeyId: 'MY_ACCESS_KEY_ID', + secretAccessKey: 'MY_SECRET_ACCESS_KEY', + }); + }, + }); +} +const credsProviderMock = jest.spyOn( + DefaultAwsCredentialsProvider.prototype, + 'getCredentials', +); + const getEntityRootDir = (entity: Entity) => { const { kind, @@ -70,7 +90,7 @@ const logger = getVoidLogger(); const loggerInfoSpy = jest.spyOn(logger, 'info'); const loggerErrorSpy = jest.spyOn(logger, 'error'); -const createPublisherFromConfig = ({ +const createPublisherFromConfig = async ({ bucketName = 'bucketName', bucketRootPath = '/', legacyUseCaseSensitiveTripletPaths = false, @@ -86,10 +106,7 @@ const createPublisherFromConfig = ({ publisher: { type: 'awsS3', awsS3: { - credentials: { - accessKeyId: 'accessKeyId', - secretAccessKey: 'secretAccessKey', - }, + accountId: '111111111111', bucketName, bucketRootPath, sse, @@ -97,9 +114,18 @@ const createPublisherFromConfig = ({ }, legacyUseCaseSensitiveTripletPaths, }, + aws: { + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'my-access-key', + secretAccessKey: 'my-secret-access-key', + }, + ], + }, }); - return AwsS3Publish.fromConfig(mockConfig, logger); + return await AwsS3Publish.fromConfig(mockConfig, logger); }; describe('AwsS3Publish', () => { @@ -151,6 +177,11 @@ describe('AwsS3Publish', () => { process.env = { ...env }; process.env.AWS_REGION = 'us-west-2'; + jest.resetAllMocks(); + credsProviderMock.mockImplementation((_?: AwsCredentialsProviderOptions) => + getMockCredentials(), + ); + mockFs({ [directory]: files, }); @@ -215,16 +246,64 @@ describe('AwsS3Publish', () => { process.env = env; }); + describe('buildCredentials', () => { + it('should retrieve credentials for a specific account ID', async () => { + await createPublisherFromConfig(); + expect(credsProviderMock).toHaveBeenCalledWith({ + accountId: '111111111111', + }); + expect(credsProviderMock).toHaveBeenCalledTimes(1); + }); + + it('should retrieve default credentials when no config is present', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + publisher: { + type: 'awsS3', + awsS3: { + bucketName: 'bucketName', + }, + }, + }, + }); + + await AwsS3Publish.fromConfig(mockConfig, logger); + expect(credsProviderMock).toHaveBeenCalledWith(); + expect(credsProviderMock).toHaveBeenCalledTimes(1); + }); + + it('should fall back to deprecated method of retrieving credentials', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + publisher: { + type: 'awsS3', + awsS3: { + credentials: { + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + }, + bucketName: 'bucketName', + bucketRootPath: '/', + }, + }, + }, + }); + + await AwsS3Publish.fromConfig(mockConfig, logger); + expect(credsProviderMock).toHaveBeenCalledTimes(0); + }); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketName: 'errorBucket', }); expect(await publisher.getReadiness()).toEqual({ @@ -235,7 +314,7 @@ describe('AwsS3Publish', () => { describe('publish', () => { it('should publish a directory', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect(await publisher.publish({ entity, directory })).toMatchObject({ objects: expect.arrayContaining([ 'default/component/backstage/404.html', @@ -246,7 +325,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory as well when legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -259,7 +338,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -272,7 +351,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -286,7 +365,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when sse is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ sse: 'aws:kms', }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -307,7 +386,7 @@ describe('AwsS3Publish', () => { 'generatedDirectory', ); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const fails = publisher.publish({ entity, @@ -327,7 +406,9 @@ describe('AwsS3Publish', () => { it('should delete stale files after upload', async () => { const bucketName = 'delete_stale_files_success'; - const publisher = createPublisherFromConfig({ bucketName: bucketName }); + const publisher = await createPublisherFromConfig({ + bucketName: bucketName, + }); await publisher.publish({ entity, directory }); expect(loggerInfoSpy).toHaveBeenLastCalledWith( `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`, @@ -336,7 +417,9 @@ describe('AwsS3Publish', () => { it('should log error when the stale files deletion fails', async () => { const bucketName = 'delete_stale_files_error'; - const publisher = createPublisherFromConfig({ bucketName: bucketName }); + const publisher = await createPublisherFromConfig({ + bucketName: bucketName, + }); await publisher.publish({ entity, directory }); expect(loggerErrorSpy).toHaveBeenLastCalledWith( 'Unable to delete file(s) from AWS S3. Error: Message', @@ -346,13 +429,13 @@ describe('AwsS3Publish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); it('should return true if docs has been generated even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -360,7 +443,7 @@ describe('AwsS3Publish', () => { }); it('should return true if docs has been generated if root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); await publisher.publish({ entity, directory }); @@ -368,7 +451,7 @@ describe('AwsS3Publish', () => { }); it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -377,7 +460,7 @@ describe('AwsS3Publish', () => { }); it('should return false if docs has not been generated', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect( await publisher.hasDocsBeenGenerated({ kind: 'entity', @@ -392,7 +475,7 @@ describe('AwsS3Publish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, @@ -400,7 +483,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -410,7 +493,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata even if root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); await publisher.publish({ entity, directory }); @@ -420,7 +503,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata if root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -442,7 +525,7 @@ describe('AwsS3Publish', () => { techdocsMetadataContent.replace(/"/g, "'"), ); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( @@ -453,7 +536,7 @@ describe('AwsS3Publish', () => { }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const invalidEntityName = { namespace: 'invalid', @@ -477,7 +560,7 @@ describe('AwsS3Publish', () => { }; }); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const invalidEntityName = { namespace: 'invalid', @@ -501,7 +584,7 @@ describe('AwsS3Publish', () => { let app: express.Express; beforeEach(async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -521,7 +604,7 @@ describe('AwsS3Publish', () => { }); it('should pass expected object path to bucket even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -541,7 +624,7 @@ describe('AwsS3Publish', () => { it('should pass expected object path to bucket if root path is specified', async () => { const rootPath = 'backstage-data/techdocs'; - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: rootPath, }); await publisher.publish({ entity, directory }); @@ -561,7 +644,7 @@ describe('AwsS3Publish', () => { it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => { const rootPath = 'backstage-data/techdocs'; - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: rootPath, legacyUseCaseSensitiveTripletPaths: true, }); diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.ts b/plugins/techdocs-node/src/stages/publish/awsS3.ts index 7c2405cc1c..c71a25cfe4 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.ts @@ -16,6 +16,10 @@ import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; +import { + AwsCredentialsProvider, + DefaultAwsCredentialsProvider, +} from '@backstage/integration-aws-node'; import { GetObjectCommand, CopyObjectCommand, @@ -27,12 +31,9 @@ import { ListObjectsV2Command, S3Client, } from '@aws-sdk/client-s3'; -import { - fromNodeProviderChain, - fromTemporaryCredentials, -} from '@aws-sdk/credential-providers'; +import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { Upload } from '@aws-sdk/lib-storage'; -import { CredentialProvider } from '@aws-sdk/types'; +import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import express from 'express'; import fs from 'fs-extra'; import JSON5 from 'json5'; @@ -97,7 +98,10 @@ export class AwsS3Publish implements PublisherBase { this.sse = options.sse; } - static fromConfig(config: Config, logger: Logger): PublisherBase { + static async fromConfig( + config: Config, + logger: Logger, + ): Promise { let bucketName = ''; try { bucketName = config.getString('techdocs.publisher.awsS3.bucketName'); @@ -121,17 +125,21 @@ export class AwsS3Publish implements PublisherBase { // or AWS shared credentials file at ~/.aws/credentials will be used. const region = config.getOptionalString('techdocs.publisher.awsS3.region'); - // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used. - // 1. AWS environment variables - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html - // 2. AWS shared credentials file at ~/.aws/credentials - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html - // 3. IAM Roles for EC2 - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html + // Credentials can optionally be configured by specifying the AWS account ID, which will retrieve credentials + // for the account from the 'aws' section of the app config. + // Credentials can also optionally be directly configured in the techdocs awsS3 config, but this method is + // deprecated. + // If no credentials are configured, the AWS SDK V3's default credential chain will be used. + const accountId = config.getOptionalString( + 'techdocs.publisher.awsS3.accountId', + ); const credentialsConfig = config.getOptionalConfig( 'techdocs.publisher.awsS3.credentials', ); - const credentials = AwsS3Publish.buildCredentials( + const credsProvider = DefaultAwsCredentialsProvider.fromConfig(config); + const credentials = await AwsS3Publish.buildCredentials( + credsProvider, + accountId, credentialsConfig, region, ); @@ -174,7 +182,7 @@ export class AwsS3Publish implements PublisherBase { private static buildStaticCredentials( accessKeyId: string, secretAccessKey: string, - ): CredentialProvider { + ): AwsCredentialIdentityProvider { return async () => { return Promise.resolve({ accessKeyId, @@ -183,20 +191,30 @@ export class AwsS3Publish implements PublisherBase { }; } - private static buildCredentials( + private static async buildCredentials( + credsProvider: AwsCredentialsProvider, + accountId?: string, config?: Config, region?: string, - ): CredentialProvider { - if (!config) { - return fromNodeProviderChain(); + ): Promise { + // Pull credentials for the specified account ID from the 'aws' config section + if (accountId) { + return (await credsProvider.getCredentials({ accountId })).provider; } + // Fall back to the default credential chain if neither account ID + // nor explicit credentials are provided + if (!config) { + return (await credsProvider.getCredentials()).provider; + } + + // Pull credentials from the techdocs config section (deprecated) const accessKeyId = config.getOptionalString('accessKeyId'); const secretAccessKey = config.getOptionalString('secretAccessKey'); - const explicitCredentials: CredentialProvider = + const explicitCredentials: AwsCredentialIdentityProvider = accessKeyId && secretAccessKey ? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey) - : fromNodeProviderChain(); + : (await credsProvider.getCredentials()).provider; const roleArn = config.getOptionalString('roleArn'); if (roleArn) { diff --git a/plugins/techdocs-node/src/stages/publish/publish.ts b/plugins/techdocs-node/src/stages/publish/publish.ts index bc7f50b5a3..2d9d07eb83 100644 --- a/plugins/techdocs-node/src/stages/publish/publish.ts +++ b/plugins/techdocs-node/src/stages/publish/publish.ts @@ -47,7 +47,7 @@ export class Publisher { return GoogleGCSPublish.fromConfig(config, logger); case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); - return AwsS3Publish.fromConfig(config, logger); + return await AwsS3Publish.fromConfig(config, logger); case 'azureBlobStorage': logger.info( 'Creating Azure Blob Storage Container publisher for TechDocs', diff --git a/yarn.lock b/yarn.lock index efdb297cbf..4041e30e7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4135,7 +4135,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/integration-aws-node@workspace:packages/integration-aws-node": +"@backstage/integration-aws-node@workspace:^, @backstage/integration-aws-node@workspace:packages/integration-aws-node": version: 0.0.0-use.local resolution: "@backstage/integration-aws-node@workspace:packages/integration-aws-node" dependencies: @@ -8142,6 +8142,7 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/integration-aws-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@google-cloud/storage": ^6.0.0 "@trendyol-js/openstack-swift-sdk": ^0.0.5