Introduce execution limiter for file uploads

* TechDoc publishers for storage providers (i.e aws, google storage)
 should now rate limit concurrent executions allowing for file uploads to run in batches of 10.
 * Add PATCH changeset to techdocs common
 * Update yarn.lock

.changeset/techdocs-dirty-rivers-hope.md

@@ -0,0 +1,5 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Add rate limiter for concurrent execution of file uploads in AWS and Google publishers

packages/techdocs-common/package.json

@@ -53,6 +53,7 @@
     "json5": "^2.1.3",
     "mime-types": "^2.1.27",
     "mock-fs": "^4.13.0",
+    "p-limit": "^3.1.0",
     "recursive-readdir": "^2.2.2",
     "winston": "^3.2.1"
   },

packages/techdocs-common/src/stages/publish/awsS3.ts

@@ -24,6 +24,7 @@ import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
 import fs from 'fs-extra';
 import { Readable } from 'stream';
 import JSON5 from 'json5';
+import limiterFactory from 'p-limit';
 
 const streamToBuffer = (stream: Readable): Promise<Buffer> => {
   return new Promise((resolve, reject) => {
@@ -131,8 +132,8 @@ export class AwsS3Publish implements PublisherBase {
       // So collecting path of only the files is good enough.
       const allFilesToUpload = await getFileTreeRecursively(directory);
 
+      const limiter = limiterFactory(10);
       const uploadPromises: Array<Promise<PutObjectCommandOutput>> = [];
-
       for (const filePath of allFilesToUpload) {
         // Remove the absolute path prefix of the source directory
         // Path of all files to upload, relative to the root of the source directory
@@ -149,7 +150,11 @@ export class AwsS3Publish implements PublisherBase {
           Body: fileContent,
         };
 
-        uploadPromises.push(this.storageClient.putObject(params));
+        // Rate limit the concurrent execution of file uploads to batches of 10
+        const uploadFile = limiter(async () =>
+          this.storageClient.putObject(params),
+        );
+        uploadPromises.push(uploadFile);
       }
       await Promise.all(uploadPromises);
       this.logger.info(

packages/techdocs-common/src/stages/publish/googleStorage.ts

@@ -26,6 +26,7 @@ import { Config } from '@backstage/config';
 import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
 import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
 import JSON5 from 'json5';
+import limitFactory from 'p-limit';
 
 export class GoogleGCSPublish implements PublisherBase {
   static async fromConfig(
@@ -102,6 +103,7 @@ export class GoogleGCSPublish implements PublisherBase {
       // So collecting path of only the files is good enough.
       const allFilesToUpload = await getFileTreeRecursively(directory);
 
+      const limiter = limitFactory(10);
       const uploadPromises: Array<Promise<UploadResponse>> = [];
       allFilesToUpload.forEach(filePath => {
         // Remove the absolute path prefix of the source directory
@@ -110,12 +112,14 @@ export class GoogleGCSPublish implements PublisherBase {
         const relativeFilePath = filePath.replace(`${directory}/`, '');
         const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
         const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path
-        // TODO: Upload in chunks of ~10 files instead of all files at once.
-        uploadPromises.push(
-          this.storageClient.bucket(this.bucketName).upload(filePath, {
-            destination,
-          }),
+
+        // Rate limit the concurrent execution of file uploads to batches of 10
+        const uploadFile = limiter(async () =>
+          this.storageClient
+            .bucket(this.bucketName)
+            .upload(filePath, { destination }),
         );
+        uploadPromises.push(uploadFile);
       });
 
       Promise.all(uploadPromises)

yarn.lock

@@ -19780,7 +19780,7 @@ p-limit@^2.0.0, p-limit@^2.2.0:
   dependencies:
     p-try "^2.0.0"
 
-p-limit@^3.0.1, p-limit@^3.0.2:
+p-limit@^3.0.1, p-limit@^3.0.2, p-limit@^3.1.0:
   version "3.1.0"
   resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
   integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==