From 225c73325806522efe27d54faa6e2bd0191915ce Mon Sep 17 00:00:00 2001 From: Bogdan Nechyporenko Date: Wed, 12 Mar 2025 19:40:46 +0100 Subject: [PATCH 1/2] Made gitlab:repo:push action idempotent. Signed-off-by: Bogdan Nechyporenko --- .changeset/good-chairs-visit.md | 5 ++ .../README.md | 63 ++++++++++++------- .../src/actions/gitlabRepoPush.ts | 52 +++++++++------ 3 files changed, 76 insertions(+), 44 deletions(-) create mode 100644 .changeset/good-chairs-visit.md diff --git a/.changeset/good-chairs-visit.md b/.changeset/good-chairs-visit.md new file mode 100644 index 0000000000..15aa04a855 --- /dev/null +++ b/.changeset/good-chairs-visit.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend-module-gitlab': patch +--- + +Made gitlab:repo:push action idempotent. diff --git a/beps/0004-scaffolder-task-idempotency/README.md b/beps/0004-scaffolder-task-idempotency/README.md index 208446158e..9c0fc7077f 100644 --- a/beps/0004-scaffolder-task-idempotency/README.md +++ b/beps/0004-scaffolder-task-idempotency/README.md @@ -154,30 +154,36 @@ export function createGithubRepoCreateAction(options: { username: owner, }); - await ctx.checkpoint('repo.creation', async () => { - const repoCreationPromise = - user.data.type === 'Organization' - ? client.rest.repos.createInOrg({ - name: repo, - org: owner, - }) - : client.rest.repos.createForAuthenticatedUser({ - name: repo, - }); - const { repoUrl } = await repoCreationPromise; - return { repoUrl }; + await ctx.checkpoint({ + key: 'repo.creation.v1', + fn: async () => { + const repoCreationPromise = + user.data.type === 'Organization' + ? client.rest.repos.createInOrg({ + name: repo, + org: owner, + }) + : client.rest.repos.createForAuthenticatedUser({ + name: repo, + }); + const { repoUrl } = await repoCreationPromise; + return { repoUrl }; + }, }); if (secrets) { - await ctx.checkpoint('repo.create.variables', async () => { - for (const [key, value] of Object.entries(repoVariables ?? {})) { - await client.rest.actions.createRepoVariable({ - owner, - repo, - name: key, - value: value, - }); - } + await ctx.checkpoint({ + key: 'repo.create.variables', + fn: async () => { + for (const [key, value] of Object.entries(repoVariables ?? {})) { + await client.rest.actions.createRepoVariable({ + owner, + repo, + name: key, + value: value, + }); + } + }, }); } @@ -202,9 +208,12 @@ Checkpoints will allow action authors to create actions where code paths are ign This will be provided on a context object and action of author provide a key and a callback. ```typescript -await ctx.checkpoint('repo.creation', async () => { - const { repoUrl } = await client.rest.Repository.create({}); - return { repoUrl }; +await ctx.checkpoint({ + key: 'repo.creation', + fn: async () => { + const { repoUrl } = await client.rest.Repository.create({}); + return { repoUrl }; + }, }); ``` @@ -259,6 +268,12 @@ Task state will be stored in the extra column `state` in the table `tasks` with } ``` +Whenever you change the return type of the checkpoint, we encourage you to change the ID. +For example, you can embed the versioning or another indicator for that. +If you'll preserve the same key, and you'll try to restart the affected task, it will fail on this checkpoint. +The cached result will not match with the expected updated return type. +By changing the key, you'll invalidate the cache of the checkpoint. + #### Workspace Persistence The workspace will be serialized and stored in the database by default. This serialization should occur at the end of a step, and after each checkpoint. It will be possible to provide additional modules to extend the workspace serialization to other providers, such as GCS or S3 instead of the database. diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabRepoPush.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabRepoPush.ts index 6537c42315..f347aaadd0 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabRepoPush.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlabRepoPush.ts @@ -151,19 +151,24 @@ export const createGitlabRepoPushAction = (options: { execute_filemode: file.executable, })); - let branchExists = false; - try { - await api.Branches.show(repoID, branchName); - branchExists = true; - } catch (e: any) { - if (e.cause?.response?.status !== 404) { - throw new InputError( - `Failed to check status of branch '${branchName}'. Please make sure that branch already exists or Backstage has permissions to create one. ${getErrorMessage( - e, - )}`, - ); - } - } + const branchExists = await ctx.checkpoint({ + key: `branch.exists.${repoID}.${branchName}`, + fn: async () => { + try { + await api.Branches.show(repoID, branchName); + return true; + } catch (e: any) { + if (e.cause?.response?.status !== 404) { + throw new InputError( + `Failed to check status of branch '${branchName}'. Please make sure that branch already exists or Backstage has permissions to create one. ${getErrorMessage( + e, + )}`, + ); + } + } + return false; + }, + }); if (!branchExists) { // create a branch using the default branch as ref @@ -181,15 +186,22 @@ export const createGitlabRepoPushAction = (options: { } try { - const commit = await api.Commits.create( - repoID, - branchName, - ctx.input.commitMessage, - actions, - ); + const commitId = await ctx.checkpoint({ + key: `commit.create.${repoID}.${branchName}`, + fn: async () => { + const commit = await api.Commits.create( + repoID, + branchName, + ctx.input.commitMessage, + actions, + ); + return commit.id; + }, + }); + ctx.output('projectid', repoID); ctx.output('projectPath', repoID); - ctx.output('commitHash', commit.id); + ctx.output('commitHash', commitId); } catch (e) { throw new InputError( `Committing the changes to ${branchName} failed. Please check that none of the files created by the template already exists. ${getErrorMessage( From b3e3315031560cd180a1519d572afaebc41f0b7e Mon Sep 17 00:00:00 2001 From: Bogdan Nechyporenko Date: Thu, 13 Mar 2025 07:38:17 +0100 Subject: [PATCH 2/2] Made gitlab:repo:push action idempotent. Signed-off-by: Bogdan Nechyporenko --- .../README.md | 6 -- .../writing-custom-actions.md | 72 ++++++++++++------- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/beps/0004-scaffolder-task-idempotency/README.md b/beps/0004-scaffolder-task-idempotency/README.md index 9c0fc7077f..e23cc519d1 100644 --- a/beps/0004-scaffolder-task-idempotency/README.md +++ b/beps/0004-scaffolder-task-idempotency/README.md @@ -268,12 +268,6 @@ Task state will be stored in the extra column `state` in the table `tasks` with } ``` -Whenever you change the return type of the checkpoint, we encourage you to change the ID. -For example, you can embed the versioning or another indicator for that. -If you'll preserve the same key, and you'll try to restart the affected task, it will fail on this checkpoint. -The cached result will not match with the expected updated return type. -By changing the key, you'll invalidate the cache of the checkpoint. - #### Workspace Persistence The workspace will be serialized and stored in the database by default. This serialization should occur at the end of a step, and after each checkpoint. It will be possible to provide additional modules to extend the workspace serialization to other providers, such as GCS or S3 instead of the database. diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 0006a52ab2..dd7a5e60a1 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -19,9 +19,11 @@ array when registering your custom actions, as seen below. ## Streamlining Custom Action Creation with Backstage CLI -The creation of custom actions in Backstage has never been easier thanks to the Backstage CLI. This tool streamlines the setup process, allowing you to focus on your actions' unique functionality. +The creation of custom actions in Backstage has never been easier thanks to the Backstage CLI. This tool streamlines the +setup process, allowing you to focus on your actions' unique functionality. -Start by using the `yarn backstage-cli new` command to generate a scaffolder module. This command sets up the necessary boilerplate code, providing a smooth start: +Start by using the `yarn backstage-cli new` command to generate a scaffolder module. This command sets up the necessary +boilerplate code, providing a smooth start: ``` $ yarn backstage-cli new @@ -34,13 +36,19 @@ $ yarn backstage-cli new You can find a [list](../../tooling/cli/03-commands.md) of all commands provided by the Backstage CLI. -When prompted, select the option to generate a scaffolder module. This creates a solid foundation for your custom action. Enter the name of the module you wish to create, and the CLI will generate the required files and directory structure. +When prompted, select the option to generate a scaffolder module. This creates a solid foundation for your custom +action. Enter the name of the module you wish to create, and the CLI will generate the required files and directory +structure. ## Writing your Custom Action -After running the command, the CLI will create a new directory with your new scaffolder module. This directory will be the working directory for creating the custom action. It will contain all the necessary files and boilerplate code to get started. +After running the command, the CLI will create a new directory with your new scaffolder module. This directory will be +the working directory for creating the custom action. It will contain all the necessary files and boilerplate code to +get started. -Let's create a simple action that adds a new file and some contents that are passed as `input` to the function. Within the generated directory, locate the file at `src/actions/example/example.ts`. Feel free to rename this file along with its generated unit test. We will replace the existing placeholder code with our custom action code as follows: +Let's create a simple action that adds a new file and some contents that are passed as `input` to the function. Within +the generated directory, locate the file at `src/actions/example/example.ts`. Feel free to rename this file along with +its generated unit test. We will replace the existing placeholder code with our custom action code as follows: ```ts title="With Zod" import { resolveSafeChildPath } from '@backstage/backend-plugin-api'; @@ -82,7 +90,8 @@ The `createTemplateAction` takes an object which specifies the following: - `id` - A unique ID for your custom action. We encourage you to namespace these in some way so that they won't collide with future built-in actions that we may ship with the `scaffolder-backend` plugin. -- `description` - An optional field to describe the purpose of the action. This will populate in the `/create/actions` endpoint. +- `description` - An optional field to describe the purpose of the action. This will populate in the `/create/actions` + endpoint. - `schema.input` - A `zod` or JSON schema object for input values to your function - `schema.output` - A `zod` or JSON schema object for values which are output from the function using `ctx.output` @@ -132,18 +141,24 @@ export const createNewFileAction = () => { ### Naming Conventions -Try to keep names consistent for both your own custom actions, and any actions contributed to open source. We've found that a separation of `:` and using a verb as the last part of the name works well. -We follow `provider:entity:verb` or as close to this as possible for our built in actions. For example, `github:actions:create` or `github:repo:create`. +Try to keep names consistent for both your own custom actions, and any actions contributed to open source. We've found +that a separation of `:` and using a verb as the last part of the name works well. +We follow `provider:entity:verb` or as close to this as possible for our built in actions. For example, +`github:actions:create` or `github:repo:create`. Also feel free to use your company name to namespace them if you prefer too, for example `acme:file:create` like above. -Prefer to use `camelCase` over `snake_case` or `kebab-case` for these actions if possible, which leads to better reading and writing of template entity definitions. +Prefer to use `camelCase` over `snake_case` or `kebab-case` for these actions if possible, which leads to better reading +and writing of template entity definitions. -> We're aware that there are some exceptions to this, but try to follow as close as possible. We'll be working on migrating these in the repository over time too. +> We're aware that there are some exceptions to this, but try to follow as close as possible. We'll be working on +> migrating these in the repository over time too. ### Adding a TemplateExample -A TemplateExample is a predefined structure that can be used to create custom actions in your software templates. It serves as a blueprint for users to understand how to use a specific action and its fields as well as to ensure consistency and standardization across different custom actions. +A TemplateExample is a predefined structure that can be used to create custom actions in your software templates. It +serves as a blueprint for users to understand how to use a specific action and its fields as well as to ensure +consistency and standardization across different custom actions. #### Define a TemplateExample and add to your Custom Action @@ -199,7 +214,8 @@ argument. It looks like the following: ## Registering Custom Actions -To register your new custom action in the Backend System you will need to create a backend module. Here is a very simplified example of how to do that: +To register your new custom action in the Backend System you will need to create a backend module. Here is a very +simplified example of how to do that: ```ts title="packages/backend/src/index.ts" /* highlight-add-start */ @@ -233,7 +249,8 @@ backend.add(import('@backstage/plugin-scaffolder-backend')); backend.add(scaffolderModuleCustomExtensions); ``` -If your custom action requires core services such as `config` or `cache` they can be imported in the dependencies and passed to the custom action function. +If your custom action requires core services such as `config` or `cache` they can be imported in the dependencies and +passed to the custom action function. ```ts title="packages/backend/src/index.ts" import { @@ -243,17 +260,17 @@ import { ... - env.registerInit({ - deps: { - scaffolder: scaffolderActionsExtensionPoint, - cache: coreServices.cache, - config: coreServices.rootConfig, - }, - async init({ scaffolder, cache, config }) { - scaffolder.addActions( - customActionNeedingCacheAndConfig({ cache: cache, config: config }), - ); - }) +env.registerInit({ + deps: { + scaffolder: scaffolderActionsExtensionPoint, + cache: coreServices.cache, + config: coreServices.rootConfig, + }, + async init({scaffolder, cache, config}) { + scaffolder.addActions( + customActionNeedingCacheAndConfig({cache: cache, config: config}), + ); + }) ``` ### Using Checkpoints in Custom Actions (Experimental) @@ -281,6 +298,13 @@ You have to define the unique key in scope of the scaffolder task for your check will check if the checkpoint with such key was already executed or not, if yes, and the run was successful, the callback will be skipped and instead the stored value will be returned. +Whenever you change the return type of the checkpoint, we encourage you to change the ID. +For example, you can embed the versioning or another indicator for that (instead of using key `create.projects`, it can +be `create.projects.v1`). +If you'll preserve the same key, and you'll try to restart the affected task, it will fail on this checkpoint. +The cached result will not match with the expected updated return type. +By changing the key, you'll invalidate the cache of the checkpoint. + ### Register Custom Actions with the Legacy Backend System Once you have your Custom Action ready for usage with the scaffolder, you'll