paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

TechDocs: upgrade to AWS SDK v3

Browse Source 
Signed-off-by: Clare Liguori <liguori@amazon.com>
This commit is contained in:
Clare Liguori
2022-11-15 10:27:51 -08:00
parent 3228a8a951
commit 37931c33ce
9 changed files with 1427 additions and 612 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/slow-flies-marry.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/plugin-events-backend-module-aws-sqs': patch
'@backstage/plugin-techdocs-node': patch
---
Upgrade to AWS SDK for Javascript v3
docs/features/techdocs/cli.md
+1 -1
View File
@@ -235,7 +235,7 @@ details.
You need to make sure that your environment is able to authenticate with the
target cloud provider. `techdocs-cli` uses the official Node.js clients provided
by AWS (v2), Google Cloud and Azure. You can authenticate using environment
by AWS (v3), Google Cloud and Azure. You can authenticate using environment
variables and/or by other means (`~/.aws/credentials`, `~/.config/gcloud` etc.)
Refer to the Authentication section of the following documentation depending
docs/features/techdocs/configuration.md
+3 -3
View File
@@ -108,8 +108,8 @@ techdocs:
# (Optional) An API key is required to write to a storage bucket.
# If not set, environment variables or aws config file will be used to authenticate.
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
# https://www.npmjs.com/package/@aws-sdk/credential-provider-node
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html
credentials:
accessKeyId: ${TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL}
secretAccessKey: ${TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL}
@@ -121,7 +121,7 @@ techdocs:
# (Optional) Endpoint URI to send requests to.
# If not set, the default endpoint is built from the configured region.
# https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
# https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html#endpoint
endpoint: ${AWS_ENDPOINT}
# (Optional) Whether to use path style URLs when communicating with S3.
docs/features/techdocs/using-cloud-storage.md
+8 -11
View File
@@ -219,22 +219,19 @@ If the environment variables
- `AWS_REGION`
are set and can be used to access the bucket you created in step 2, they will be
used by the AWS SDK V2 Node.js client for authentication.
[Refer to the official documentation for loading credentials in Node.js from environment variables](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html).
used by the AWS SDK V3 Node.js client for authentication.
[Refer to the official documentation for loading credentials in Node.js from environment variables](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html).
If the environment variables are missing, the AWS SDK tries to read the
`~/.aws/credentials` file for credentials.
[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html)
[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html)
If you are using Amazon EC2 instance to deploy Backstage, you do not need to
obtain the access keys separately. They can be made available in the environment
automatically by defining appropriate IAM role with access to the bucket. Read
more in
If you are deploying Backstage to Amazon EC2, Amazon ECS, or Amazon EKS, you do
not need to obtain the access keys separately. They can be made available in the
environment automatically by defining appropriate IAM role with access to the
bucket. Read more in the
[official AWS documentation for using IAM roles.](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html#use-roles).
The AWS Region of the bucket is optional since TechDocs uses AWS SDK V2 and not
V3.
**4b. Authentication using app-config.yaml**
AWS credentials and region can be provided to the AWS SDK via `app-config.yaml`.
@@ -254,7 +251,7 @@ techdocs:
```
Refer to the
[official AWS documentation for obtaining the credentials](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/getting-your-credentials.html).
[official AWS documentation for obtaining the credentials](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html).
**4c. Authentication using an assumed role** Users with multiple AWS accounts
may want to use a role for S3 storage that is in a different AWS account. Using
plugins/events-backend-module-aws-sqs/package.json
+2 -1
View File
@@ -23,7 +23,7 @@
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@aws-sdk/client-sqs": "^3.0.0",
"@aws-sdk/client-sqs": "^3.208.0",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/backend-tasks": "workspace:^",
"@backstage/config": "workspace:^",
@@ -33,6 +33,7 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@aws-sdk/types": "^3.208.0",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
plugins/techdocs-node/package.json
+5 -1
View File
@@ -39,6 +39,10 @@
"url": "https://github.com/backstage/backstage/issues"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.208.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/lib-storage": "^3.208.0",
"@aws-sdk/types": "^3.208.0",
"@azure/identity": "^2.0.1",
"@azure/storage-blob": "^12.5.0",
"@backstage/backend-common": "workspace:^",
@@ -50,7 +54,6 @@
"@google-cloud/storage": "^6.0.0",
"@trendyol-js/openstack-swift-sdk": "^0.0.5",
"@types/express": "^4.17.6",
"aws-sdk": "^2.840.0",
"express": "^4.17.1",
"fs-extra": "10.1.0",
"git-url-parse": "^13.0.0",
@@ -70,6 +73,7 @@
"@types/mock-fs": "^4.13.0",
"@types/recursive-readdir": "^2.2.0",
"@types/supertest": "^2.0.8",
"aws-sdk-client-mock": "^2.0.0",
"supertest": "^6.1.3"
}
}
plugins/techdocs-node/src/stages/publish/awsS3.test.ts
+130 -104
View File
@@ -14,119 +14,31 @@
* limitations under the License.
*/
import {
DeleteObjectCommand,
GetObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
S3Client,
UploadPartCommand
} from '@aws-sdk/client-s3';
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { mockClient, AwsClientStub } from 'aws-sdk-client-mock';
import express from 'express';
import request from 'supertest';
import mockFs from 'mock-fs';
import path from 'path';
import fs, { ReadStream } from 'fs-extra';
import { EventEmitter } from 'events';
import fs from 'fs-extra';
import { AwsS3Publish } from './awsS3';
import { storageRootDir } from '../../testUtils/StorageFilesMock';
import { Readable } from 'stream';
jest.mock('aws-sdk', () => {
const { StorageFilesMock } = require('../../testUtils/StorageFilesMock');
const storage = new StorageFilesMock();
return {
__esModule: true,
Credentials: jest.requireActual('aws-sdk').Credentials,
default: {
S3: class {
constructor() {
storage.emptyFiles();
}
headObject({ Key }: { Key: string }) {
return {
promise: async () => {
if (!storage.fileExists(Key)) {
throw new Error('File does not exist');
}
},
};
}
getObject({ Key }: { Key: string }) {
return {
promise: async () => storage.fileExists(Key),
createReadStream: () => {
const emitter = new EventEmitter();
process.nextTick(() => {
if (storage.fileExists(Key)) {
emitter.emit('data', Buffer.from(storage.readFile(Key)));
emitter.emit('end');
} else {
emitter.emit(
'error',
new Error(`The file ${Key} does not exist!`),
);
}
});
return emitter;
},
};
}
headBucket({ Bucket }: { Bucket: string }) {
return {
promise: async () => {
if (Bucket === 'errorBucket') {
throw new Error('Bucket does not exist');
}
return {};
},
};
}
upload({ Key, Body }: { Key: string; Body: ReadStream }) {
return {
promise: () =>
new Promise(async resolve => {
const chunks = new Array<Buffer>();
Body.on('data', chunk => {
chunks.push(chunk as Buffer);
});
Body.once('end', () => {
storage.writeFile(Key, Buffer.concat(chunks));
resolve(null);
});
}),
};
}
listObjectsV2({ Bucket }: { Bucket: string }) {
return {
promise: () => {
if (
Bucket === 'delete_stale_files_success' ||
Bucket === 'delete_stale_files_error'
) {
return Promise.resolve({
Contents: [{ Key: 'stale_file.png' }],
});
}
return Promise.resolve({});
},
};
}
deleteObject({ Bucket }: { Bucket: string }) {
return {
promise: () => {
if (Bucket === 'delete_stale_files_error') {
throw new Error('Message');
}
return Promise.resolve();
},
};
}
},
},
};
});
const env = process.env;
let s3Mock: AwsClientStub<S3Client>;
const getEntityRootDir = (entity: Entity) => {
const {
@@ -137,6 +49,23 @@ const getEntityRootDir = (entity: Entity) => {
return path.join(storageRootDir, namespace || DEFAULT_NAMESPACE, kind, name);
};
class ErrorReadable extends Readable {
errorMessage: string;
constructor(errorMessage: string) {
super();
this.errorMessage = errorMessage;
}
_read() {
this.destroy(new Error(this.errorMessage));
}
_destroy(error: Error | null, callback: (error?: Error | null) => void) {
callback(error);
}
}
const logger = getVoidLogger();
const loggerInfoSpy = jest.spyOn(logger, 'info');
const loggerErrorSpy = jest.spyOn(logger, 'error');
@@ -219,13 +148,71 @@ describe('AwsS3Publish', () => {
};
beforeEach(() => {
process.env = { ...env };
process.env.AWS_REGION = 'us-west-2';
mockFs({
[directory]: files,
});
const { StorageFilesMock } = require('../../testUtils/StorageFilesMock');
const storage = new StorageFilesMock();
storage.emptyFiles();
s3Mock = mockClient(S3Client);
s3Mock.on(HeadObjectCommand).callsFake(input => {
if (!storage.fileExists(input.Key)) {
throw new Error('File does not exist');
}
return {};
});
s3Mock.on(GetObjectCommand).callsFake(input => {
if (storage.fileExists(input.Key)) {
return {
Body: Readable.from(storage.readFile(input.Key)),
};
}
throw new Error(`The file ${input.Key} does not exist!`);
});
s3Mock.on(HeadBucketCommand).callsFake(input => {
if (input.Bucket === 'errorBucket') {
throw new Error('Bucket does not exist');
}
return {};
});
s3Mock.on(ListObjectsV2Command).callsFake(input => {
if (
input.Bucket === 'delete_stale_files_success' ||
input.Bucket === 'delete_stale_files_error'
) {
return {
Contents: [{ Key: 'stale_file.png' }],
};
}
return {};
});
s3Mock.on(DeleteObjectCommand).callsFake(input => {
if (input.Bucket === 'delete_stale_files_error') {
throw new Error('Message');
}
return {};
});
s3Mock.on(UploadPartCommand).rejects();
s3Mock.on(PutObjectCommand).callsFake(input => {
storage.writeFile(input.Key, input.Body);
});
});
afterEach(() => {
mockFs.restore();
process.env = env;
});
describe('getReadiness', () => {
@@ -478,7 +465,30 @@ describe('AwsS3Publish', () => {
await expect(fails).rejects.toMatchObject({
message: expect.stringMatching(
'TechDocs metadata fetch failed; caused by Error: Unable to read stream; caused by Error: The file invalid/triplet/path/techdocs_metadata.json does not exist',
'TechDocs metadata fetch failed; caused by Error: The file invalid/triplet/path/techdocs_metadata.json does not exist',
),
});
});
it('should return an error if the techdocs_metadata.json file cannot be read from stream', async () => {
s3Mock.on(GetObjectCommand).callsFake(_ => {
return {
Body: new ErrorReadable('No stream!'),
}});
const publisher = createPublisherFromConfig();
const invalidEntityName = {
namespace: 'invalid',
kind: 'triplet',
name: 'path',
};
const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
await expect(fails).rejects.toMatchObject({
message: expect.stringMatching(
'TechDocs metadata fetch failed; caused by Error: Unable to read stream; caused by Error: No stream!',
),
});
});
@@ -597,5 +607,21 @@ describe('AwsS3Publish', () => {
'File Not Found',
);
});
it('should return 404 if file cannot be read from stream', async () => {
s3Mock.on(GetObjectCommand).callsFake(_ => {
return {
Body: new ErrorReadable('No stream!'),
}});
const response = await request(app).get(
`/${entityTripletPath}/not-found.html`,
);
expect(response.status).toBe(404);
expect(Buffer.from(response.text).toString('utf8')).toEqual(
'File Not Found',
);
});
});
});
plugins/techdocs-node/src/stages/publish/awsS3.ts
+95 -64
View File
@@ -16,9 +16,23 @@
import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { assertError, ForwardedError } from '@backstage/errors';
import aws, { Credentials } from 'aws-sdk';
import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
import {
GetObjectCommand,
CopyObjectCommand,
DeleteObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
PutObjectCommandInput,
ListObjectsV2CommandOutput,
ListObjectsV2Command,
S3Client,
} from '@aws-sdk/client-s3';
import {
fromNodeProviderChain,
fromTemporaryCredentials,
} from '@aws-sdk/credential-providers';
import { Upload } from '@aws-sdk/lib-storage';
import { CredentialProvider } from '@aws-sdk/types';
import express from 'express';
import fs from 'fs-extra';
import JSON5 from 'json5';
@@ -60,7 +74,7 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
};
export class AwsS3Publish implements PublisherBase {
private readonly storageClient: aws.S3;
private readonly storageClient: S3Client;
private readonly bucketName: string;
private readonly legacyPathCasing: boolean;
private readonly logger: Logger;
@@ -68,7 +82,7 @@ export class AwsS3Publish implements PublisherBase {
private readonly sse?: 'aws:kms' | 'AES256';
constructor(options: {
storageClient: aws.S3;
storageClient: S3Client;
bucketName: string;
legacyPathCasing: boolean;
logger: Logger;
@@ -103,6 +117,10 @@ export class AwsS3Publish implements PublisherBase {
| 'AES256'
| undefined;
// AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
// or AWS shared credentials file at ~/.aws/credentials will be used.
const region = config.getOptionalString('techdocs.publisher.awsS3.region');
// Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
// 1. AWS environment variables
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
@@ -113,11 +131,7 @@ export class AwsS3Publish implements PublisherBase {
const credentialsConfig = config.getOptionalConfig(
'techdocs.publisher.awsS3.credentials',
);
const credentials = AwsS3Publish.buildCredentials(credentialsConfig);
// AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
// or AWS shared credentials file at ~/.aws/credentials will be used.
const region = config.getOptionalString('techdocs.publisher.awsS3.region');
const credentials = AwsS3Publish.buildCredentials(credentialsConfig, region);
// AWS endpoint is an optional config. If missing, the default endpoint is built from
// the configured region.
@@ -131,8 +145,9 @@ export class AwsS3Publish implements PublisherBase {
'techdocs.publisher.awsS3.s3ForcePathStyle',
);
const storageClient = new aws.S3({
credentials,
const storageClient = new S3Client({
customUserAgent: 'backstage-aws-techdocs-s3-publisher',
credentialDefaultProvider: () => credentials,
...(region && { region }),
...(endpoint && { endpoint }),
...(s3ForcePathStyle && { s3ForcePathStyle }),
@@ -153,31 +168,39 @@ export class AwsS3Publish implements PublisherBase {
});
}
private static buildCredentials(
config?: Config,
): Credentials | CredentialsOptions | undefined {
private static buildStaticCredentials(
accessKeyId: string,
secretAccessKey: string,
): CredentialProvider {
return async () => {
return Promise.resolve({
accessKeyId,
secretAccessKey,
});
};
}
private static buildCredentials(config?: Config, region?: string): CredentialProvider {
if (!config) {
return undefined;
return fromNodeProviderChain();
}
const accessKeyId = config.getOptionalString('accessKeyId');
const secretAccessKey = config.getOptionalString('secretAccessKey');
let explicitCredentials: Credentials | undefined;
if (accessKeyId && secretAccessKey) {
explicitCredentials = new Credentials({
accessKeyId,
secretAccessKey,
});
}
const explicitCredentials: CredentialProvider =
accessKeyId && secretAccessKey
? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey)
: fromNodeProviderChain();
const roleArn = config.getOptionalString('roleArn');
if (roleArn) {
return new aws.ChainableTemporaryCredentials({
return fromTemporaryCredentials({
masterCredentials: explicitCredentials,
params: {
RoleSessionName: 'backstage-aws-techdocs-s3-publisher',
RoleArn: roleArn,
},
clientConfig: { region },
});
}
@@ -190,9 +213,9 @@ export class AwsS3Publish implements PublisherBase {
*/
async getReadiness(): Promise<ReadinessResponse> {
try {
await this.storageClient
.headBucket({ Bucket: this.bucketName })
.promise();
await this.storageClient.send(
new HeadBucketCommand({ Bucket: this.bucketName }),
);
this.logger.info(
`Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
@@ -258,7 +281,7 @@ export class AwsS3Publish implements PublisherBase {
const relativeFilePath = path.relative(directory, absoluteFilePath);
const fileStream = fs.createReadStream(absoluteFilePath);
const params = {
const params: PutObjectCommandInput = {
Bucket: this.bucketName,
Key: getCloudPathForLocalPath(
entity,
@@ -268,10 +291,15 @@ export class AwsS3Publish implements PublisherBase {
),
Body: fileStream,
...(sse && { ServerSideEncryption: sse }),
} as aws.S3.PutObjectRequest;
};
objects.push(params.Key);
return this.storageClient.upload(params).promise();
objects.push(params.Key!);
const upload = new Upload({
client: this.storageClient,
params,
});
return upload.done();
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 },
@@ -301,12 +329,12 @@ export class AwsS3Publish implements PublisherBase {
await bulkStorageOperation(
async relativeFilePath => {
return await this.storageClient
.deleteObject({
return await this.storageClient.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: relativeFilePath,
})
.promise();
}),
);
},
staleFiles,
{ concurrencyLimit: 10 },
@@ -334,15 +362,17 @@ export class AwsS3Publish implements PublisherBase {
const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
const stream = this.storageClient
.getObject({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
})
.createReadStream();
try {
const techdocsMetadataJson = await streamToBuffer(stream);
const resp = await this.storageClient.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`,
}),
);
const techdocsMetadataJson = await streamToBuffer(
resp.Body as Readable,
);
if (!techdocsMetadataJson) {
throw new Error(
`Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
@@ -384,10 +414,11 @@ export class AwsS3Publish implements PublisherBase {
const fileExtension = path.extname(filePath);
const responseHeaders = getHeadersForFileExtension(fileExtension);
const stream = this.storageClient
.getObject({ Bucket: this.bucketName, Key: filePath })
.createReadStream();
try {
const resp = await this.storageClient.send(
new GetObjectCommand({ Bucket: this.bucketName, Key: filePath }),
);
// Inject response headers
for (const [headerKey, headerValue] of Object.entries(
responseHeaders,
@@ -395,7 +426,7 @@ export class AwsS3Publish implements PublisherBase {
res.setHeader(headerKey, headerValue);
}
res.send(await streamToBuffer(stream));
res.send(await streamToBuffer(resp.Body as Readable));
} catch (err) {
assertError(err);
this.logger.warn(
@@ -419,12 +450,12 @@ export class AwsS3Publish implements PublisherBase {
const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);
await this.storageClient
.headObject({
await this.storageClient.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`,
})
.promise();
}),
);
return Promise.resolve(true);
} catch (e) {
return Promise.resolve(false);
@@ -457,21 +488,21 @@ export class AwsS3Publish implements PublisherBase {
try {
this.logger.verbose(`Migrating ${file}`);
await this.storageClient
.copyObject({
await this.storageClient.send(
new CopyObjectCommand({
Bucket: this.bucketName,
CopySource: [this.bucketName, file].join('/'),
Key: newPath,
})
.promise();
}),
);
if (removeOriginal) {
await this.storageClient
.deleteObject({
await this.storageClient.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: file,
})
.promise();
}),
);
}
} catch (e) {
assertError(e);
@@ -490,16 +521,16 @@ export class AwsS3Publish implements PublisherBase {
): Promise<string[]> {
const objects: string[] = [];
let nextContinuation: string | undefined;
let allObjects: ListObjectsV2Output;
let allObjects: ListObjectsV2CommandOutput;
// Iterate through every file in the root of the publisher.
do {
allObjects = await this.storageClient
.listObjectsV2({
allObjects = await this.storageClient.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
ContinuationToken: nextContinuation,
...(prefix ? { Prefix: prefix } : {}),
})
.promise();
}),
);
objects.push(
...(allObjects.Contents || []).map(f => f.Key || '').filter(f => !!f),
);
yarn.lock
+1177 -427
View File
File diff suppressed because it is too large Load Diff