Scaffolder workspace serialization

Signed-off-by: bnechyporenko <bnechyporenko@bol.com>

.changeset/healthy-dots-ring.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-scaffolder-node': patch
+---
+
+Scaffolder workspace serialization

plugins/scaffolder-backend/api-report.md

@@ -3,6 +3,8 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+/// <reference types="node" />
+
 import { ActionContext as ActionContext_2 } from '@backstage/plugin-scaffolder-node';
 import { AuthService } from '@backstage/backend-plugin-api';
 import * as azure from '@backstage/plugin-scaffolder-backend-module-azure';
@@ -370,6 +372,8 @@ export interface CurrentClaimedTask {
   spec: TaskSpec;
   state?: JsonObject;
   taskId: string;
+  // (undocumented)
+  workspace?: Promise<Buffer>;
 }
 
 // @public
@@ -414,6 +418,8 @@ export class DatabaseTaskStore implements TaskStore {
     | undefined
   >;
   // (undocumented)
+  getWorkspace?(options: { taskId: string }): Promise<Buffer | undefined>;
+  // (undocumented)
   heartbeatTask(taskId: string): Promise<void>;
   // (undocumented)
   list(options: { createdBy?: string }): Promise<{
@@ -437,6 +443,8 @@ export class DatabaseTaskStore implements TaskStore {
   // (undocumented)
   saveTaskState(options: { taskId: string; state?: JsonObject }): Promise<void>;
   // (undocumented)
+  serializeWorkspace(options: { path: string; taskId: string }): Promise<void>;
+  // (undocumented)
   shutdownTask(options: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
@@ -554,10 +562,14 @@ export class TaskManager implements TaskContext_2 {
     | undefined
   >;
   // (undocumented)
+  getWorkspace(options: { taskId: string }): Promise<Buffer | undefined>;
+  // (undocumented)
   getWorkspaceName(): Promise<string>;
   // (undocumented)
   get secrets(): TaskSecrets_2 | undefined;
   // (undocumented)
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+  // (undocumented)
   get spec(): TaskSpecV1beta3;
   // (undocumented)
   updateCheckpoint?(
@@ -609,6 +621,8 @@ export interface TaskStore {
     | undefined
   >;
   // (undocumented)
+  getWorkspace?(options: { taskId: string }): Promise<Buffer | undefined>;
+  // (undocumented)
   heartbeatTask(taskId: string): Promise<void>;
   // (undocumented)
   list?(options: { createdBy?: string }): Promise<{
@@ -634,6 +648,14 @@ export interface TaskStore {
     state?: JsonObject;
   }): Promise<void>;
   // (undocumented)
+  serializeWorkspace?({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
+  // (undocumented)
   shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 

plugins/scaffolder-backend/migrations/20240401213200_workspace.js

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.binary('workspace').nullable().comment('A snapshot of the workspace');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.dropColumn('workspace');
+  });
+};

plugins/scaffolder-backend/package.json

@@ -78,6 +78,7 @@
     "@backstage/types": "workspace:^",
     "@types/express": "^4.17.6",
     "@types/luxon": "^3.0.0",
+    "concat-stream": "^2.0.0",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "fs-extra": "^11.2.0",
@@ -93,6 +94,7 @@
     "p-limit": "^3.1.0",
     "p-queue": "^6.6.2",
     "prom-client": "^15.0.0",
+    "tar": "^6.1.12",
     "uuid": "^9.0.0",
     "winston": "^3.2.1",
     "winston-transport": "^4.7.0",

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -42,6 +42,7 @@ import { DateTime, Duration } from 'luxon';
 import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common';
 import { trimEventsTillLastRecovery } from './taskRecoveryHelper';
 import { intervalFromNowTill } from './dbUtil';
+import { restoreWorkspace, serializeWorkspace } from './serializer';
 
 const migrationsDir = resolvePackagePath(
   '@backstage/plugin-scaffolder-backend',
@@ -57,6 +58,7 @@ export type RawDbTaskRow = {
   created_at: string;
   created_by: string | null;
   secrets?: string | null;
+  workspace?: Buffer;
 };
 
 export type RawDbTaskEventRow = {
@@ -509,6 +511,30 @@ export class DatabaseTaskStore implements TaskStore {
     });
   }
 
+  async rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void> {
+    const [result] = await this.db<RawDbTaskRow>('tasks')
+      .where({ id: options.taskId })
+      .select('workspace');
+
+    await restoreWorkspace(options.targetPath, result.workspace);
+  }
+
+  async serializeWorkspace(options: {
+    path: string;
+    taskId: string;
+  }): Promise<void> {
+    if (options.path) {
+      await this.db<RawDbTaskRow>('tasks')
+        .where({ id: options.taskId })
+        .update({
+          workspace: await serializeWorkspace(options.path),
+        });
+    }
+  }
+
   async cancelTask(
     options: TaskStoreEmitOptions<{ message: string } & JsonObject>,
   ): Promise<void> {

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -419,6 +419,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
                 reason: stringifyError(err),
               });
               throw err;
+            } finally {
+              await task.serializeWorkspace?.({ path: workspacePath });
             }
           },
           createTemporaryDirectory: async () => {
@@ -460,6 +462,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
       await taskTrack.markFailed(step, err);
       await stepTrack.markFailed();
       throw err;
+    } finally {
+      await task.serializeWorkspace?.({ path: workspacePath });
     }
   }
 
@@ -469,10 +473,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
         'Wrong template version executed with the workflow engine',
       );
     }
-    const workspacePath = path.join(
-      this.options.workingDirectory,
-      await task.getWorkspaceName(),
-    );
+    const taskId = await task.getWorkspaceName();
+
+    const workspacePath = path.join(this.options.workingDirectory, taskId);
 
     const { additionalTemplateFilters, additionalTemplateGlobals } =
       this.options;
@@ -486,6 +489,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     });
 
     try {
+      await task.rehydrateWorkspace?.({ taskId, targetPath: workspacePath });
+
       const taskTrack = await this.tracker.taskStart(task);
       await fs.ensureDir(workspacePath);
 

plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts

@@ -99,6 +99,13 @@ export class TaskManager implements TaskContext {
     return this.task.taskId;
   }
 
+  async rehydrateWorkspace(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void> {
+    return this.storage.rehydrateWorkspace?.(options);
+  }
+
   get done() {
     return this.isDone;
   }
@@ -144,6 +151,13 @@ export class TaskManager implements TaskContext {
     });
   }
 
+  async serializeWorkspace?(options: { path: string }): Promise<void> {
+    await this.storage.serializeWorkspace?.({
+      path: options.path,
+      taskId: this.task.taskId,
+    });
+  }
+
   async complete(
     result: TaskCompletionState,
     metadata?: JsonObject,
@@ -219,6 +233,8 @@ export interface CurrentClaimedTask {
    * The creator of the task.
    */
   createdBy?: string;
+
+  workspace?: Promise<Buffer>;
 }
 
 function defer() {

plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { serializeWorkspace, restoreWorkspace } from './serializer';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import fs from 'fs-extra';
+
+describe('serializer', () => {
+  const workspaceDir = createMockDirectory({
+    content: {
+      'app-config.yaml': `
+            app:
+              title: Example App
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config2.yaml': `
+            app:
+              title: Example App 2
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config.development.yaml': `
+            app:
+              sessionKey: development-key
+            backend:
+              $include: ./included.yaml
+            other:
+              $include: secrets/included.yaml
+          `,
+      'secrets/session-key.txt': 'abc123',
+      'secrets/included.yaml': `
+            secret:
+              $file: session-key.txt
+          `,
+      'included.yaml': `
+            foo:
+              bar: token \${MY_SECRET}
+          `,
+      'app-config.substitute.yaml': `
+            app:
+              someConfig:
+                $include: \${SUBSTITUTE_ME}.yaml
+              noSubstitute:
+                $file: \$\${ESCAPE_ME}.txt
+          `,
+      'substituted.yaml': `
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `,
+      'secrets/substituted.txt': '123abc',
+      '${ESCAPE_ME}.txt': 'notSubstituted',
+      'empty.yaml': '# just a comment',
+    },
+  });
+
+  const restoredWorkspaceDir = createMockDirectory();
+
+  it('should be able to archive and restore the workspace', async () => {
+    const workspaceBuffer = await serializeWorkspace(workspaceDir.path);
+    await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer);
+
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`),
+    ).toBeTruthy();
+    expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`),
+    ).toBeTruthy();
+
+    expect(
+      fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'),
+    ).toEqual(`
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `);
+  });
+});

plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import tar from 'tar';
+import concatStream from 'concat-stream';
+import { promisify } from 'util';
+import { pipeline as pipelineCb, Readable } from 'stream';
+
+const pipeline = promisify(pipelineCb);
+
+export const serializeWorkspace = async (path: string): Promise<Buffer> => {
+  return await new Promise<Buffer>(async resolve => {
+    await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve));
+  });
+};
+
+export const restoreWorkspace = async (path: string, buffer?: Buffer) => {
+  if (buffer) {
+    await pipeline(
+      Readable.from(buffer),
+      tar.extract({
+        C: path,
+      }),
+    );
+  }
+};

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -211,6 +211,19 @@ export interface TaskStore {
   ): Promise<{ events: SerializedTaskEvent[] }>;
 
   shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise<void>;
+
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+
+  serializeWorkspace?({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
 }
 
 export type WorkflowResponse = { output: { [key: string]: JsonValue } };

plugins/scaffolder-node/api-report.md

@@ -361,12 +361,16 @@ export interface TaskContext {
     | undefined
   >;
   // (undocumented)
+  getWorkspace?(options: { taskId: string }): Promise<Buffer | undefined>;
+  // (undocumented)
   getWorkspaceName(): Promise<string>;
   // (undocumented)
   isDryRun?: boolean;
   // (undocumented)
   secrets?: TaskSecrets;
   // (undocumented)
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+  // (undocumented)
   spec: TaskSpec;
   // (undocumented)
   updateCheckpoint?(

plugins/scaffolder-node/src/tasks/types.ts

@@ -141,6 +141,13 @@ export interface TaskContext {
         },
   ): Promise<void>;
 
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+
   getWorkspaceName(): Promise<string>;
 
   getInitiatorCredentials(): Promise<BackstageCredentials>;

yarn.lock

@@ -6788,6 +6788,7 @@ __metadata:
     "@types/nunjucks": ^3.1.4
     "@types/supertest": ^2.0.8
     "@types/zen-observable": ^0.8.0
+    concat-stream: ^2.0.0
     esbuild: ^0.20.0
     express: ^4.17.1
     express-promise-router: ^4.1.0
@@ -6806,6 +6807,7 @@ __metadata:
     prom-client: ^15.0.0
     strip-ansi: ^7.1.0
     supertest: ^6.1.3
+    tar: ^6.1.12
     uuid: ^9.0.0
     wait-for-expect: ^3.0.2
     winston: ^3.2.1