From ea2bbef1b16d95b87bbdb9a10d74cb5db7959270 Mon Sep 17 00:00:00 2001 From: Jos Craw Date: Wed, 22 Feb 2023 21:52:03 +1300 Subject: [PATCH 1/3] feat: add support for HTTPS proxy for AWS S3 requests in Techdocs Signed-off-by: Jos Craw --- .changeset/wicked-lions-repeat.md | 6 ++++++ packages/techdocs-cli/cli-report.md | 1 + packages/techdocs-cli/src/commands/index.ts | 4 ++++ packages/techdocs-cli/src/lib/PublisherConfig.ts | 1 + plugins/techdocs-node/package.json | 2 ++ plugins/techdocs-node/src/stages/publish/awsS3.ts | 12 ++++++++++++ yarn.lock | 4 +++- 7 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .changeset/wicked-lions-repeat.md diff --git a/.changeset/wicked-lions-repeat.md b/.changeset/wicked-lions-repeat.md new file mode 100644 index 0000000000..c48545c6bc --- /dev/null +++ b/.changeset/wicked-lions-repeat.md @@ -0,0 +1,6 @@ +--- +'@techdocs/cli': minor +'@backstage/plugin-techdocs-node': minor +--- + +Added support for an HTTPS proxy for techdocs AWS S3 requests diff --git a/packages/techdocs-cli/cli-report.md b/packages/techdocs-cli/cli-report.md index 846ccc88ca..556068de85 100644 --- a/packages/techdocs-cli/cli-report.md +++ b/packages/techdocs-cli/cli-report.md @@ -77,6 +77,7 @@ Options: --azureAccountKey --awsRoleArn --awsEndpoint + --awsProxy --awsS3sse --awsS3ForcePathStyle --awsBucketRootPath diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index 102e6d87de..98816a6b32 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -173,6 +173,10 @@ export function registerCommands(program: Command) { '--awsEndpoint ', 'Optional AWS endpoint to send requests to.', ) + .option( + '--awsProxy ', + 'Optional Proxy to use for AWS requests.', + ) .option('--awsS3sse ', 'Optional AWS S3 Server Side Encryption.') .option( '--awsS3ForcePathStyle', diff --git a/packages/techdocs-cli/src/lib/PublisherConfig.ts b/packages/techdocs-cli/src/lib/PublisherConfig.ts index 0ef6b66eaf..803e40308b 100644 --- a/packages/techdocs-cli/src/lib/PublisherConfig.ts +++ b/packages/techdocs-cli/src/lib/PublisherConfig.ts @@ -94,6 +94,7 @@ export class PublisherConfig { ...(opts.awsEndpoint && { endpoint: opts.awsEndpoint }), ...(opts.awsS3ForcePathStyle && { s3ForcePathStyle: true }), ...(opts.awsS3sse && { sse: opts.awsS3sse }), + ...(opts.awsProxy && { httpsProxy: opts.awsProxy }), }, }; } diff --git a/plugins/techdocs-node/package.json b/plugins/techdocs-node/package.json index 4f458c4fff..2b9e74a359 100644 --- a/plugins/techdocs-node/package.json +++ b/plugins/techdocs-node/package.json @@ -42,6 +42,7 @@ "@aws-sdk/client-s3": "^3.208.0", "@aws-sdk/credential-providers": "^3.208.0", "@aws-sdk/lib-storage": "^3.208.0", + "@aws-sdk/node-http-handler": "^3.208.0", "@aws-sdk/types": "^3.208.0", "@azure/identity": "^2.1.0", "@azure/storage-blob": "^12.5.0", @@ -58,6 +59,7 @@ "express": "^4.17.1", "fs-extra": "10.1.0", "git-url-parse": "^13.0.0", + "hpagent": "^1.2.0", "js-yaml": "^4.0.0", "json5": "^2.1.3", "mime-types": "^2.1.27", diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.ts b/plugins/techdocs-node/src/stages/publish/awsS3.ts index 6b528cfd30..679e9bf7c9 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.ts @@ -32,8 +32,10 @@ import { S3Client, } from '@aws-sdk/client-s3'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; +import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; import { Upload } from '@aws-sdk/lib-storage'; import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; +import { HttpsProxyAgent } from 'hpagent'; import express from 'express'; import fs from 'fs-extra'; import JSON5 from 'json5'; @@ -150,6 +152,11 @@ export class AwsS3Publish implements PublisherBase { 'techdocs.publisher.awsS3.endpoint', ); + // AWS HTTPS proxy is an optional config. If missing, no proxy is used + const httpsProxy = config.getOptionalString( + 'techdocs.publisher.awsS3.httpsProxy', + ); + // AWS forcePathStyle is an optional config. If missing, it defaults to false. Needs to be enabled for cases // where endpoint url points to locally hosted S3 compatible storage like Localstack const forcePathStyle = config.getOptionalBoolean( @@ -162,6 +169,11 @@ export class AwsS3Publish implements PublisherBase { ...(region && { region }), ...(endpoint && { endpoint }), ...(forcePathStyle && { forcePathStyle }), + ...(httpsProxy && { + requestHandler: new NodeHttpHandler({ + httpsAgent: new HttpsProxyAgent({ proxy: httpsProxy }), + }), + }), }); const legacyPathCasing = diff --git a/yarn.lock b/yarn.lock index 544b7beec4..934bf0d7e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1211,7 +1211,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/node-http-handler@npm:3.272.0": +"@aws-sdk/node-http-handler@npm:3.272.0, @aws-sdk/node-http-handler@npm:^3.208.0": version: 3.272.0 resolution: "@aws-sdk/node-http-handler@npm:3.272.0" dependencies: @@ -8516,6 +8516,7 @@ __metadata: "@aws-sdk/client-s3": ^3.208.0 "@aws-sdk/credential-providers": ^3.208.0 "@aws-sdk/lib-storage": ^3.208.0 + "@aws-sdk/node-http-handler": ^3.208.0 "@aws-sdk/types": ^3.208.0 "@azure/identity": ^2.1.0 "@azure/storage-blob": ^12.5.0 @@ -8540,6 +8541,7 @@ __metadata: express: ^4.17.1 fs-extra: 10.1.0 git-url-parse: ^13.0.0 + hpagent: ^1.2.0 js-yaml: ^4.0.0 json5: ^2.1.3 mime-types: ^2.1.27 From cc77729c48d040dfe108a8c3599cac9fe4ceb946 Mon Sep 17 00:00:00 2001 From: Jos Craw Date: Sat, 25 Feb 2023 08:45:00 +1300 Subject: [PATCH 2/3] fix: update docs to include updated command reference Signed-off-by: Jos Craw --- docs/features/techdocs/cli.md | 45 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/features/techdocs/cli.md b/docs/features/techdocs/cli.md index 682c6a8f8f..6b2e4383b8 100644 --- a/docs/features/techdocs/cli.md +++ b/docs/features/techdocs/cli.md @@ -176,29 +176,28 @@ Usage: techdocs-cli publish [options] Publish generated TechDocs site to an external storage AWS S3, Google GCS, etc. Options: - --publisher-type (Required always) awsS3 | googleGcs | azureBlobStorage - - same as techdocs.publisher.type in Backstage - app-config.yaml - --storage-name (Required always) In case of AWS/GCS, use the bucket - name. In case of Azure, use container name. Same as - techdocs.publisher.[TYPE].bucketName - --entity (Required always) Entity uid separated by / in - namespace/kind/name order (case-sensitive). Example: - default/Component/myEntity - --legacyUseCaseSensitiveTripletPaths Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name). - Only use if your TechDocs backend is configured the same way - --azureAccountName (Required for Azure) specify when --publisher-type - azureBlobStorage - --azureAccountKey Azure Storage Account key to use for authentication. - If not specified, you must set AZURE_TENANT_ID, - AZURE_CLIENT_ID & AZURE_CLIENT_SECRET as environment - variables. - --awsRoleArn Optional AWS ARN of role to be assumed. - --awsEndpoint Optional AWS endpoint to send requests to. - --awsS3ForcePathStyle Optional AWS S3 option to force path style. - --directory Path of the directory containing generated files to - publish (default: "./site/") - -h, --help display help for command + --publisher-type (Required always) awsS3 | googleGcs | azureBlobStorage | openStackSwift - same as techdocs.publisher.type in Backstage app-config.yaml + --storage-name (Required always) In case of AWS/GCS, use the bucket name. In case of Azure, use container name. Same as + techdocs.publisher.[TYPE].bucketName + --entity (Required always) Entity uid separated by / in namespace/kind/name order (case-sensitive). Example: default/Component/myEntity + --legacyUseCaseSensitiveTripletPaths Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name). Only use if your TechDocs backend is configured + the same way. (default: false) + --azureAccountName (Required for Azure) specify when --publisher-type azureBlobStorage + --azureAccountKey Azure Storage Account key to use for authentication. If not specified, you must set AZURE_TENANT_ID, AZURE_CLIENT_ID & + AZURE_CLIENT_SECRET as environment variables. + --awsRoleArn Optional AWS ARN of role to be assumed. + --awsEndpoint Optional AWS endpoint to send requests to. + --awsProxy Optional Proxy to use for AWS requests. + --awsS3sse Optional AWS S3 Server Side Encryption. + --awsS3ForcePathStyle Optional AWS S3 option to force path style. + --awsBucketRootPath Optional sub-directory to store files in Amazon S3 + --osCredentialId (Required for OpenStack) specify when --publisher-type openStackSwift + --osSecret (Required for OpenStack) specify when --publisher-type openStackSwift + --osAuthUrl (Required for OpenStack) specify when --publisher-type openStackSwift + --osSwiftUrl (Required for OpenStack) specify when --publisher-type openStackSwift + --gcsBucketRootPath Optional sub-directory to store files in Google cloud storage + --directory Path of the directory containing generated files to publish (default: "./site/") + -h, --help display help for command ``` ### Migrate content for case-insensitive access From 8b36144ab00804cd5f29523313c7405a0b812f10 Mon Sep 17 00:00:00 2001 From: Jos Craw Date: Sun, 26 Feb 2023 10:47:18 +1300 Subject: [PATCH 3/3] chore: added docs for overall config Signed-off-by: Jos Craw --- docs/features/techdocs/configuration.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index e95db2fad0..49ee9d72c2 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -136,6 +136,11 @@ techdocs: # https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html#endpoint endpoint: ${AWS_ENDPOINT} + # (Optional) HTTPS proxy to use for S3 Requests + # Defaults to using no proxy + # This allows docs to be published and read from behind a proxy + httpsProxy: ${HTTPS_PROXY} + # (Optional) Whether to use path style URLs when communicating with S3. # Defaults to false. # This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used to host tech docs.