Replace S3 read-store-upload loop with file streams

This no longer buffers file contents in memory and limits the maximum
number of file descriptors open concurrently.

Signed-off-by: Joel Low <joel@joelsplace.sg>

.changeset/young-pens-argue.md

@@ -0,0 +1,5 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Remove read-store-upload loop when uploading S3 objects for TechDocs

packages/techdocs-common/src/stages/publish/awsS3.test.ts

@@ -99,6 +99,9 @@ describe('AwsS3Publish', () => {
           assets: {
             'main.css': '',
           },
+          attachments: {
+            'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]),
+          },
         },
       });
     });

packages/techdocs-common/src/stages/publish/awsS3.ts

@@ -167,18 +167,18 @@ export class AwsS3Publish implements PublisherBase {
         const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
         const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path
 
-        const fileContent = await fs.readFile(filePath, 'utf8');
-
-        const params = {
-          Bucket: this.bucketName,
-          Key: destination,
-          Body: fileContent,
-        };
-
         // Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
-        const uploadFile = limiter(() =>
-          this.storageClient.upload(params).promise(),
-        );
+        const uploadFile = limiter(() => {
+          const fileStream = fs.createReadStream(filePath);
+
+          const params = {
+            Bucket: this.bucketName,
+            Key: destination,
+            Body: fileStream,
+          };
+
+          return this.storageClient.upload(params).promise();
+        });
         uploadPromises.push(uploadFile);
       }
       await Promise.all(uploadPromises);