From c777df180afc3a6bede6d7d7febc0dea1902b396 Mon Sep 17 00:00:00 2001 From: vitorgrenzel Date: Wed, 13 Jan 2021 09:20:58 -0300 Subject: [PATCH] feat(techdocs-common): add Azure Storage --- .changeset/chilly-dodos-drop.md | 6 + app-config.yaml | 2 +- docs/features/techdocs/README.md | 12 +- docs/features/techdocs/configuration.md | 13 + docs/features/techdocs/using-cloud-storage.md | 61 +++++ .../__mocks__/@azure/storage-blob.ts | 86 +++++++ packages/techdocs-common/package.json | 1 + .../src/stages/publish/azureStorage.test.ts | 149 ++++++++++++ .../src/stages/publish/azureStorage.ts | 223 ++++++++++++++++++ .../src/stages/publish/publish.test.ts | 25 ++ .../src/stages/publish/publish.ts | 4 + .../src/stages/publish/types.ts | 2 +- .../techdocs-backend/src/service/router.ts | 1 + plugins/techdocs/config.d.ts | 39 +++ 14 files changed, 616 insertions(+), 8 deletions(-) create mode 100644 .changeset/chilly-dodos-drop.md create mode 100644 packages/techdocs-common/__mocks__/@azure/storage-blob.ts create mode 100644 packages/techdocs-common/src/stages/publish/azureStorage.test.ts create mode 100644 packages/techdocs-common/src/stages/publish/azureStorage.ts diff --git a/.changeset/chilly-dodos-drop.md b/.changeset/chilly-dodos-drop.md new file mode 100644 index 0000000000..b262c14e4d --- /dev/null +++ b/.changeset/chilly-dodos-drop.md @@ -0,0 +1,6 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs-backend': patch +--- + +1. Added option to use Azure Storage as a choice to store the static generated files for TechDocs. diff --git a/app-config.yaml b/app-config.yaml index c150ef5902..09d2e75253 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -80,7 +80,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureStorage'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index c43d6d978d..24b4410092 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -108,12 +108,12 @@ providers are used. | GitLab | Yes ✅ | | GitLab Enterprise | Yes ✅ | -| File Storage Provider | Support Status | -| --------------------------------- | ----------------------------------------------------------------- | -| Local Filesystem of Backstage app | Yes ✅ | -| Google Cloud Storage (GCS) | Yes ✅ | -| Amazon Web Services (AWS) S3 | Yes ✅ | -| Azure Storage | No ❌ [#3938](https://github.com/backstage/backstage/issues/3938) | +| File Storage Provider | Support Status | +| --------------------------------- | -------------- | +| Local Filesystem of Backstage app | Yes ✅ | +| Google Cloud Storage (GCS) | Yes ✅ | +| Amazon Web Services (AWS) S3 | Yes ✅ | +| Azure Storage | Yes ✅ | [Reach out to us](#feedback) if you want to request more platforms. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 1580abe69e..8d5906b18f 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -84,4 +84,17 @@ techdocs: # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html region: $env: AWS_REGION + + # Required when techdocs.publisher.type is set to 'azureStorage'. Skip otherwise. + + azureStorage: + # An API key is required to write to a storage container. + credentials: + account: + $env: TECHDOCS_AZURE_STORAGE_ACCOUNT + accountKey: + $env: TECHDOCS_AZURE_STORAGE_ACCOUNT_KEY + + # Azure Storage Container Name + containerName: 'techdocs-storage' ``` diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 21206dede5..2d906ca1b9 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -195,3 +195,64 @@ Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read the static generated documentation files. When you start the backend of the app, you should be able to see `techdocs info Successfully connected to the AWS S3 bucket` in the logs. + +## Configuring Azure Storage Container with TechDocs + +Follow the +[official Azure Storage documentation](https://docs.microsoft.com/pt-br/javascript/api/@azure/storage-blob/?view=azure-node-latest) +for the latest instructions on the following steps involving Azure Storage. + +**1. Set `techdocs.publisher.type` config in your `app-config.yaml`** + +Set `techdocs.publisher.type` to `'azureStorage'`. + +```yaml +techdocs: + publisher: + type: 'azureStorage' +``` + +**2. Service account credentials** + +To get credentials, access the Azure Portal and go to "Settings > Access Keys", +and get your Storage account name and Primary Key. + +```yaml +techdocs: + publisher: + type: 'azureStorage' + azureStorage: + credentials: + account: 'account' + accountKey: 'accountKey' +``` + +**3. Azure Storage Container** + +Create a dedicated container for TechDocs sites. techdocs-backend will publish +documentation to this container. TechDocs will fetch files from here to serve +documentation in Backstage. + +To create a new container, access "Blob Service > Containers > New Container". + +Set the name of the container to +`techdocs.publisher.azureStorage.containerName`. + +```yaml +techdocs: + publisher: + type: 'azureStorage' + azureStorage: + credentials: + account: 'account' + accountKey: 'accountKey' + containerName: 'name-of-techdocs-storage-container' +``` + +**4. That's it!** + +Your Backstage app is now ready to use Azure Storage for TechDocs, to store and +read the static generated documentation files. When you start the backend of the +app, you should be able to see +`techdocs info Successfully connected to the Azure Storage container` in the +logs. diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts new file mode 100644 index 0000000000..60a705f6a7 --- /dev/null +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs'; + +export class BlockBlobClient { + private readonly blobName; + + constructor(blobName: string) { + this.blobName = blobName; + } + + uploadFile(source: string) { + return new Promise((resolve, reject) => { + if (!fs.existsSync(source)) { + reject(''); + } else { + resolve(''); + } + }); + } + + exists() { + return new Promise((resolve, reject) => { + if (fs.existsSync(this.blobName)) { + resolve(true); + } else { + reject({ message: 'The object doest not exist !' }); + } + }); + } +} + +export class ContainerClient { + private readonly containerName; + + constructor(containerName: string) { + this.containerName = containerName; + } + + getProperties() { + return new Promise(resolve => { + resolve(''); + }); + } + + getBlockBlobClient(blobName: string) { + return new BlockBlobClient(blobName); + } +} + +export class BlobServiceClient { + private readonly url; + private readonly credential; + + constructor(url: string, credential?: StorageSharedKeyCredential) { + this.url = url; + this.credential = credential; + } + + getContainerClient(containerName: string) { + return new ContainerClient(containerName); + } +} + +export class StorageSharedKeyCredential { + private readonly accountName; + private readonly accountKey; + + constructor(accountName: string, accountKey: string) { + this.accountName = accountName; + this.accountKey = accountKey; + } +} diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 64c5c19425..09893bd38e 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -36,6 +36,7 @@ "url": "https://github.com/backstage/backstage/issues" }, "dependencies": { + "@azure/storage-blob": "^12.3.0", "@aws-sdk/client-s3": "^3.1.0", "@backstage/backend-common": "^0.5.1", "@backstage/catalog-model": "^0.7.0", diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureStorage.test.ts new file mode 100644 index 0000000000..05b5441bff --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/azureStorage.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { AzureStoragePublish } from './azureStorage'; +import { PublisherBase } from './types'; +import type { Entity } from '@backstage/catalog-model'; + +const createMockEntity = (annotations = {}) => { + return { + apiVersion: 'version', + kind: 'TestKind', + metadata: { + name: 'test-component-name', + namespace: 'test-namespace', + annotations: { + ...annotations, + }, + }, + }; +}; + +const getEntityRootDir = (entity: Entity) => { + const { + kind, + metadata: { namespace, name }, + } = entity; + const entityRootDir = `${namespace}/${kind}/${name}`; + return entityRootDir; +}; + +const logger = getVoidLogger(); +jest.spyOn(logger, 'info').mockReturnValue(logger); +jest.spyOn(logger, 'error').mockReturnValue(logger); + +let publisher: PublisherBase; + +beforeEach(async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureStorage', + azureStorage: { + credentials: { + account: 'account', + accountKey: 'accountKey', + }, + containerName: 'containerName', + }, + }, + }, + }); + + publisher = await AzureStoragePublish.fromConfig(mockConfig, logger); +}); + +describe('AzureStoragePublish', () => { + describe('publish', () => { + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + expect( + await publisher.publish({ + entity, + directory: entityRootDir, + }), + ).toBeUndefined(); + mockFs.restore(); + }); + + it('should fail to publish a directory', async () => { + const wrongPathToGeneratedDirectory = 'wrong/path/to/generatedDirectory'; + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + await publisher + .publish({ + entity, + directory: wrongPathToGeneratedDirectory, + }) + .catch(error => + expect(error).toEqual( + new Error( + `Unable to upload file(s) to Azure Storage. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`, + ), + ), + ); + mockFs.restore(); + }); + }); + + describe('hasDocsBeenGenerated', () => { + it('should return true if docs has been generated', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': 'file-content', + }, + }); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + mockFs.restore(); + }); + + it('should return false if docs has not been generated', async () => { + const entity = createMockEntity(); + + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + }); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/azureStorage.ts b/packages/techdocs-common/src/stages/publish/azureStorage.ts new file mode 100644 index 0000000000..bc9a6825d9 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/azureStorage.ts @@ -0,0 +1,223 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import express from 'express'; +import { + BlobServiceClient, + BlobUploadCommonResponse, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Logger } from 'winston'; +import { Entity, EntityName } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; +import { PublisherBase, PublishRequest } from './types'; + +export class AzureStoragePublish implements PublisherBase { + static async fromConfig( + config: Config, + logger: Logger, + ): Promise { + let account = ''; + let accountKey = ''; + let containerName = ''; + try { + account = config.getString( + 'techdocs.publisher.azureStorage.credentials.account', + ); + accountKey = config.getString( + 'techdocs.publisher.azureStorage.credentials.accountKey', + ); + containerName = config.getString( + 'techdocs.publisher.azureStorage.containerName', + ); + } catch (error) { + throw new Error( + "Since techdocs.publisher.type is set to 'azureStorage' in your app config, " + + 'credentials and containerName are required in techdocs.publisher.azureStorage ' + + 'required to authenticate with Azure Storage.', + ); + } + + const credential = new StorageSharedKeyCredential(account, accountKey); + const storageClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + credential, + ); + + await storageClient + .getContainerClient(containerName) + .getProperties() + .then(() => { + logger.info( + `Successfully connected to the Azure Storage container ${containerName}.`, + ); + }) + .catch(reason => { + logger.error( + `Could not retrieve metadata about the Azure Storage container ${containerName}. ` + + 'Make sure the Azure project and the container exists and the access key located at the path ' + + "techdocs.publisher.azureStorage.credentials defined in app config has the role 'Storage Object Creator'. " + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + throw new Error(`from Azure Storage client library: ${reason.message}`); + }); + + return new AzureStoragePublish(storageClient, containerName, logger); + } + + constructor( + private readonly storageClient: BlobServiceClient, + private readonly containerName: string, + private readonly logger: Logger, + ) { + this.storageClient = storageClient; + this.containerName = containerName; + this.logger = logger; + } + + /** + * Upload all the files from the generated `directory` to the Azure Storage container. + * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html + */ + async publish({ entity, directory }: PublishRequest): Promise { + try { + // Note: Azure Storage manages creation of parent directories if they do not exist. + // So collecting path of only the files is good enough. + const allFilesToUpload = await getFileTreeRecursively(directory); + + const uploadPromises: Array> = []; + allFilesToUpload.forEach(filePath => { + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + const relativeFilePath = filePath.replace(`${directory}/`, ''); + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = path.normalize( + `${entityRootDir}/${relativeFilePath}`, + ); // Azure Storage Container file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. + uploadPromises.push( + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(destination) + .uploadFile(filePath), + ); + }); + + await Promise.all(uploadPromises).then(() => { + this.logger.info( + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, + ); + }); + return; + } catch (e) { + const errorMessage = `Unable to upload file(s) to Azure Storage. Error ${e.message}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + } + + download(containerName: string, path: string): Promise { + return new Promise((resolve, reject) => { + const fileStreamChunks: Array = []; + this.storageClient + .getContainerClient(containerName) + .getBlockBlobClient(path) + .download() + .then(res => { + const body = res.readableStreamBody; + if (!body) { + reject(new Error(`Unable to parse the response data`)); + return; + } + body + .on('error', e => { + this.logger.error(e.message); + reject(e.message); + }) + .on('data', chunk => { + fileStreamChunks.push(chunk); + }) + .on('end', () => { + resolve(Buffer.concat(fileStreamChunks).toString()); + }); + }); + }); + } + + async fetchTechDocsMetadata(entityName: EntityName): Promise { + const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + try { + return this.download( + this.containerName, + `${entityRootDir}/techdocs_metadata.json`, + ); + } catch (e) { + this.logger.error(e.message); + throw e; + } + } + + /** + * Express route middleware to serve static files on a route in techdocs-backend. + */ + docsRouter(): express.Handler { + return (req, res) => { + // Trim the leading forward slash + // filePath example - /default/Component/documented-component/index.html + const filePath = req.path.replace(/^\//, ''); + // Files with different extensions (CSS, HTML) need to be served with different headers + const fileExtension = path.extname(filePath); + const responseHeaders = getHeadersForFileExtension(fileExtension); + + try { + this.download(this.containerName, filePath).then(fileContent => { + // Inject response headers + for (const [headerKey, headerValue] of Object.entries( + responseHeaders, + )) { + res.setHeader(headerKey, headerValue); + } + res.send(fileContent); + }); + } catch (e) { + this.logger.error(e.message); + res.status(404).send(e.message); + } + }; + } + + /** + * A helper function which checks if index.html of an Entity's docs site is available. This + * can be used to verify if there are any pre-generated docs available to serve. + */ + async hasDocsBeenGenerated(entity: Entity): Promise { + return new Promise(resolve => { + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + this.storageClient + .getContainerClient(this.containerName) + .getBlockBlobClient(`${entityRootDir}/index.html`) + .exists() + .then((response: boolean) => { + resolve(response); + }) + .catch(() => { + resolve(false); + }); + }); + } +} diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index 89f2c0ffdd..a97eadb404 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -22,6 +22,7 @@ import { Publisher } from './publish'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; +import { AzureStoragePublish } from './azureStorage'; const logger = getVoidLogger(); const discovery: jest.Mocked = { @@ -105,4 +106,28 @@ describe('Publisher', () => { }); expect(publisher).toBeInstanceOf(AwsS3Publish); }); + + it('should create Azure Storage publisher from config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureStorage', + azureStorage: { + credentials: { + account: 'account', + accountKey: 'accountKey', + }, + containerName: 'containerName', + }, + }, + }, + }); + + const publisher = await Publisher.fromConfig(mockConfig, { + logger, + discovery, + }); + expect(publisher).toBeInstanceOf(AzureStoragePublish); + }); }); diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index 82232c2fd1..fc72d2812e 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -21,6 +21,7 @@ import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; import { GoogleGCSPublish } from './googleStorage'; import { AwsS3Publish } from './awsS3'; +import { AzureStoragePublish } from './azureStorage'; type factoryOptions = { logger: Logger; @@ -47,6 +48,9 @@ export class Publisher { case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); + case 'azureStorage': + logger.info('Creating Azure Storage Container publisher for TechDocs'); + return AzureStoragePublish.fromConfig(config, logger); case 'local': logger.info('Creating Local publisher for TechDocs'); return new LocalPublish(config, logger, discovery); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index db6a075d43..f67f65ee23 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -19,7 +19,7 @@ import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. */ -export type PublisherType = 'local' | 'googleGcs' | 'awsS3'; +export type PublisherType = 'local' | 'googleGcs' | 'awsS3' | 'azureStorage'; export type PublishRequest = { entity: Entity; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 5ad23cde84..cbb6efa3dd 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -148,6 +148,7 @@ export async function createRouter({ } break; case 'awsS3': + case 'azureStorage': case 'googleGcs': // This block should be valid for all external storage implementations. So no need to duplicate in future, // add the publisher type in the list here. diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index f9831f27cb..1dd7319f61 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -114,6 +114,45 @@ export interface Config { region?: string; }; } + | { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'azureStorage' + * alternatives: 'azureStorage' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: 'azureStorage'; + + /** + * azureStorage required when 'type' is set to azureStorage + */ + azureStorage?: { + /** + * Credentials used to access a storage container + * @visibility secret + */ + credentials: { + /** + * Account access name + * attr: 'account' - accepts a string value + * @visibility secret + */ + account: string; + /** + * Account secret primary key + * attr: 'accountKey' - accepts a string value + * @visibility secret + */ + accountKey: string; + }; + /** + * Cloud Storage Container Name + * attr: 'containerName' - accepts a string value + * @visibility backend + */ + containerName: string; + }; + } | { /** * attr: 'type' - accepts a string value