paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

Merge pull request #6322 from backstage/mob/nunjucks-renderer

Browse Source 
scaffolder: Add a node-based templating action
This commit is contained in:
Ben Lambert
2021-07-08 16:51:51 +02:00
committed by GitHub
parent a43e5d1de6 eece4003bf
commit df12cc25aa
16 changed files with 654 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/scaffolder-backend/api-report.md
+6
View File
@@ -74,6 +74,12 @@ export function createFetchPlainAction(options: {
integrations: ScmIntegrations;
}): TemplateAction<any>;
// @public (undocumented)
export function createFetchTemplateAction(options: {
reader: UrlReader;
integrations: ScmIntegrations;
}): TemplateAction<any>;
// @public (undocumented)
export const createFilesystemDeleteAction: () => TemplateAction<any>;
plugins/scaffolder-backend/fixtures/test-nested-template/public/react-logo192.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.2 KiB

plugins/scaffolder-backend/package.json
+3
View File
@@ -51,12 +51,14 @@
"globby": "^11.0.0",
"handlebars": "^4.7.6",
"helmet": "^4.0.0",
"isbinaryfile": "^4.0.8",
"isomorphic-git": "^1.8.0",
"jsonschema": "^1.2.6",
"knex": "^0.95.1",
"lodash": "^4.17.21",
"luxon": "^1.26.0",
"morgan": "^1.10.0",
"nunjucks": "^3.2.3",
"octokit-plugin-create-pull-request": "^3.9.3",
"uuid": "^8.2.0",
"winston": "^3.2.1",
@@ -69,6 +71,7 @@
"@types/fs-extra": "^9.0.1",
"@types/git-url-parse": "^9.0.0",
"@types/mock-fs": "^4.13.0",
"@types/nunjucks": "^3.1.4",
"@types/supertest": "^2.0.8",
"jest-when": "^3.1.0",
"mock-fs": "^4.13.0",
plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts
+9 -1
View File
@@ -24,7 +24,11 @@ import {
} from './catalog';
import { createDebugLogAction } from './debug';
import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch';
import {
createFetchCookiecutterAction,
createFetchPlainAction,
createFetchTemplateAction,
} from './fetch';
import {
createFilesystemDeleteAction,
createFilesystemRenameAction,
@@ -62,6 +66,10 @@ export const createBuiltinActions = (options: {
integrations,
containerRunner,
}),
createFetchTemplateAction({
integrations,
reader,
}),
createPublishGithubAction({
integrations,
config,
plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts
+1 -1
View File
@@ -26,7 +26,7 @@ export function createDebugLogAction() {
return createTemplateAction<{ message?: string; listWorkspace?: boolean }>({
id: 'debug:log',
description:
'Writes a message into the log or list all files in the workspace.',
'Writes a message into the log or lists all files in the workspace.',
schema: {
input: {
type: 'object',
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts
+1 -1
View File
@@ -136,7 +136,7 @@ export function createFetchCookiecutterAction(options: {
}>({
id: 'fetch:cookiecutter',
description:
'Downloads a template from the given URL into the workspace, and runs cookiecutter on it.',
"Downloads a template from the given URL into the workspace, and runs cookiecutter on it. This action is deprecated in favor of 'fetch:template'. See https://backstage.io/docs/features/software-templates/builtin-actions#migrating-from-fetch-cookiecutter-to-fetch-template for more details.",
schema: {
input: {
type: 'object',
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts
+1
View File
@@ -16,4 +16,5 @@
export { createFetchPlainAction } from './plain';
export { createFetchCookiecutterAction } from './cookiecutter';
export { createFetchTemplateAction } from './template';
export { fetchContents } from './helpers';
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts
+295
View File
@@ -0,0 +1,295 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import { join as joinPath, resolve as resolvePath } from 'path';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { ScmIntegrations } from '@backstage/integration';
import { PassThrough } from 'stream';
import { fetchContents } from './helpers';
import { ActionContext, TemplateAction } from '../../types';
import { createFetchTemplateAction, FetchTemplateInput } from './template';
jest.mock('./helpers', () => ({
fetchContents: jest.fn(),
}));
const aBinaryFile = fs.readFileSync(
resolvePath(
'src',
'../fixtures/test-nested-template/public/react-logo192.png',
),
);
const mockFetchContents = fetchContents as jest.MockedFunction<
typeof fetchContents
>;
describe('fetch:template', () => {
let action: TemplateAction<any>;
const workspacePath = os.tmpdir();
const createTemporaryDirectory: jest.MockedFunction<
ActionContext<FetchTemplateInput>['createTemporaryDirectory']
> = jest.fn(() =>
Promise.resolve(
joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`),
),
);
const logger = getVoidLogger();
const mockContext = (inputPatch: Partial<FetchTemplateInput> = {}) => ({
baseUrl: 'base-url',
input: {
url: './skeleton',
targetPath: './target',
values: {
test: 'value',
},
...inputPatch,
},
output: jest.fn(),
logStream: new PassThrough(),
logger,
workspacePath,
createTemporaryDirectory,
});
beforeEach(() => {
mockFs();
action = createFetchTemplateAction({
reader: (Symbol('UrlReader') as unknown) as UrlReader,
integrations: (Symbol('Integrations') as unknown) as ScmIntegrations,
});
});
afterEach(() => {
mockFs.restore();
});
it(`returns a TemplateAction with the id 'fetch:template'`, () => {
expect(action.id).toEqual('fetch:template');
});
describe('handler', () => {
it('throws if output directory is outside the workspace', async () => {
await expect(() =>
action.handler(mockContext({ targetPath: '../' })),
).rejects.toThrowError(
/relative path is not allowed to refer to a directory outside its parent/i,
);
});
it('throws if copyWithoutRender parameter is not an array', async () => {
await expect(() =>
action.handler(
mockContext({ copyWithoutRender: ('abc' as unknown) as string[] }),
),
).rejects.toThrowError(/copyWithoutRender must be an array/i);
});
describe('with valid input', () => {
let context: ActionContext<FetchTemplateInput>;
beforeEach(async () => {
context = mockContext({
values: {
name: 'test-project',
count: 1234,
itemList: ['first', 'second', 'third'],
},
});
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
[outputPath]: {
'empty-dir-${{ values.count }}': {},
'static.txt': 'static content',
'${{ values.name }}.txt': 'static content',
subdir: {
'templated-content.txt':
'${{ values.name }}: ${{ values.count }}',
},
'.${{ values.name }}': '${{ values.itemList | dump }}',
'a-binary-file.png': aBinaryFile,
},
});
return Promise.resolve();
});
await action.handler(context);
});
it('uses fetchContents to retrieve the template content', () => {
expect(mockFetchContents).toHaveBeenCalledWith(
expect.objectContaining({
baseUrl: context.baseUrl,
fetchUrl: context.input.url,
}),
);
});
it('copies files with no templating in names or content successfully', async () => {
await expect(
fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'),
).resolves.toEqual('static content');
});
it('copies files with templated names successfully', async () => {
await expect(
fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
).resolves.toEqual('static content');
});
it('copies files with templated content successfully', async () => {
await expect(
fs.readFile(
`${workspacePath}/target/subdir/templated-content.txt`,
'utf-8',
),
).resolves.toEqual('test-project: 1234');
});
it('processes dotfiles', async () => {
await expect(
fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'),
).resolves.toEqual('["first","second","third"]');
});
it('copies empty directories', async () => {
await expect(
fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'),
).resolves.toEqual([]);
});
it('copies binary files as-is without processing them', async () => {
await expect(
fs.readFile(`${workspacePath}/target/a-binary-file.png`),
).resolves.toEqual(aBinaryFile);
});
});
describe('copyWithoutRender', () => {
let context: ActionContext<FetchTemplateInput>;
beforeEach(async () => {
context = mockContext({
values: {
name: 'test-project',
count: 1234,
},
copyWithoutRender: ['.unprocessed'],
});
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
[outputPath]: {
processed: {
'templated-content-${{ values.name }}.txt':
'${{ values.count }}',
},
'.unprocessed': {
'templated-content-${{ values.name }}.txt':
'${{ values.count }}',
},
},
});
return Promise.resolve();
});
await action.handler(context);
});
it('ignores template syntax in files matched in copyWithoutRender', async () => {
await expect(
fs.readFile(
`${workspacePath}/target/.unprocessed/templated-content-\${{ values.name }}.txt`,
'utf-8',
),
).resolves.toEqual('${{ values.count }}');
});
it('processes files not matched in copyWithoutRender', async () => {
await expect(
fs.readFile(
`${workspacePath}/target/processed/templated-content-test-project.txt`,
'utf-8',
),
).resolves.toEqual('1234');
});
});
});
describe('cookiecutter compatibility mode', () => {
let context: ActionContext<FetchTemplateInput>;
beforeEach(async () => {
context = mockContext({
values: {
name: 'test-project',
count: 1234,
itemList: ['first', 'second', 'third'],
},
cookiecutterCompat: true,
});
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
[outputPath]: {
'{{ cookiecutter.name }}.txt': 'static content',
subdir: {
'templated-content.txt':
'{{ cookiecutter.name }}: {{ cookiecutter.count }}',
},
'{{ cookiecutter.name }}.json':
'{{ cookiecutter.itemList | jsonify }}',
},
});
return Promise.resolve();
});
await action.handler(context);
});
it('copies files with cookiecutter-style templated names successfully', async () => {
await expect(
fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
).resolves.toEqual('static content');
});
it('copies files with cookiecutter-style templated content successfully', async () => {
await expect(
fs.readFile(
`${workspacePath}/target/subdir/templated-content.txt`,
'utf-8',
),
).resolves.toEqual('test-project: 1234');
});
it('includes the jsonify filter', async () => {
await expect(
fs.readFile(`${workspacePath}/target/test-project.json`, 'utf-8'),
).resolves.toEqual('["first","second","third"]');
});
});
});
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts
+239
View File
@@ -0,0 +1,239 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolve as resolvePath } from 'path';
import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common';
import { InputError } from '@backstage/errors';
import { ScmIntegrations } from '@backstage/integration';
import { fetchContents } from './helpers';
import { createTemplateAction } from '../../createTemplateAction';
import globby from 'globby';
import nunjucks from 'nunjucks';
import fs from 'fs-extra';
import { isBinaryFile } from 'isbinaryfile';
/*
* Maximise compatibility with Jinja (and therefore cookiecutter)
* using nunjucks jinja compat mode. Since this method mutates
* the global nunjucks instance, we can't enable this per-template,
* or only for templates with cookiecutter compat enabled, so the
* next best option is to explicitly enable it globally and allow
* folks to rely on jinja compatibility behaviour in fetch:template
* templates if they wish.
*
* cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat
*/
nunjucks.installJinjaCompat();
export type FetchTemplateInput = {
url: string;
targetPath?: string;
values: any;
copyWithoutRender?: string[];
cookiecutterCompat?: boolean;
};
export function createFetchTemplateAction(options: {
reader: UrlReader;
integrations: ScmIntegrations;
}) {
const { reader, integrations } = options;
return createTemplateAction<FetchTemplateInput>({
id: 'fetch:template',
description:
"Downloads a skeleton, templates variables into file and directory names and content, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.",
schema: {
input: {
type: 'object',
required: ['url'],
properties: {
url: {
title: 'Fetch URL',
description:
'Relative path or absolute URL pointing to the directory tree to fetch',
type: 'string',
},
targetPath: {
title: 'Target Path',
description:
'Target path within the working directory to download the contents to. Defaults to the working directory root.',
type: 'string',
},
values: {
title: 'Template Values',
description: 'Values to pass on to the templating engine',
type: 'object',
},
copyWithoutRender: {
title: 'Copy Without Render',
description:
'An array of glob patterns. Any files or directories which match are copied without being processed as templates.',
type: 'array',
items: {
type: 'string',
},
},
cookiecutterCompat: {
title: 'Cookiecutter compatibility mode',
description:
'Enable features to maximise compatibility with templates built for fetch:cookiecutter',
type: 'boolean',
},
},
},
},
async handler(ctx) {
ctx.logger.info('Fetching template content from remote URL');
const workDir = await ctx.createTemporaryDirectory();
const templateDir = resolvePath(workDir, 'template');
const targetPath = ctx.input.targetPath ?? './';
const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath);
if (
ctx.input.copyWithoutRender &&
!Array.isArray(ctx.input.copyWithoutRender)
) {
throw new InputError(
'Fetch action input copyWithoutRender must be an Array',
);
}
await fetchContents({
reader,
integrations,
baseUrl: ctx.baseUrl,
fetchUrl: ctx.input.url,
outputPath: templateDir,
});
ctx.logger.info('Listing files and directories in template');
const allEntriesInTemplate = await globby(`**/*`, {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
});
const nonTemplatedEntries = new Set(
(
await Promise.all(
(ctx.input.copyWithoutRender || []).map(pattern =>
globby(pattern, {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
}),
),
)
).flat(),
);
// Create a templater
const templater = nunjucks.configure({
...(ctx.input.cookiecutterCompat
? {}
: {
tags: {
// TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR?
variableStart: '${{',
variableEnd: '}}',
},
}),
// We don't want this builtin auto-escaping, since uses HTML escape sequences
// like `&quot;` - the correct way to escape strings in our case depends on
// the file type.
autoescape: false,
});
if (ctx.input.cookiecutterCompat) {
// The "jsonify" filter built into cookiecutter is common
// in fetch:cookiecutter templates, so when compat mode
// is enabled we alias the "dump" filter from nunjucks as
// jsonify. Dump accepts an optional `spaces` parameter
// which enables indented output, but when this parameter
// is not supplied it works identically to jsonify.
//
// cf. https://cookiecutter.readthedocs.io/en/latest/advanced/template_extensions.html?highlight=jsonify#jsonify-extension
// cf. https://mozilla.github.io/nunjucks/templating.html#dump
templater.addFilter('jsonify', templater.getFilter('dump'));
}
// Cookiecutter prefixes all parameters in templates with
// `cookiecutter.`. To replicate this, we wrap our parameters
// in an object with a `cookiecutter` property when compat
// mode is enabled.
const { cookiecutterCompat, values } = ctx.input;
const context = {
[cookiecutterCompat ? 'cookiecutter' : 'values']: values,
};
ctx.logger.info(
`Processing ${allEntriesInTemplate.length} template files/directories with input values`,
ctx.input.values,
);
for (const location of allEntriesInTemplate) {
const shouldCopyWithoutRender = nonTemplatedEntries.has(location);
const outputPath = resolvePath(
outputDir,
shouldCopyWithoutRender
? location
: templater.renderString(location, context),
);
if (shouldCopyWithoutRender) {
ctx.logger.info(
`Copying file/directory ${location} without processing since it matches a pattern in "copyWithoutRender".`,
);
}
if (location.endsWith('/')) {
ctx.logger.info(
`Writing directory ${location} to template output path.`,
);
await fs.ensureDir(outputPath);
} else {
const inputFilePath = resolvePath(templateDir, location);
if (await isBinaryFile(inputFilePath)) {
ctx.logger.info(
`Copying binary file ${location} to template output path.`,
);
await fs.copy(inputFilePath, outputPath);
} else {
ctx.logger.info(
`Writing file ${location} to template output path.`,
);
const inputFileContents = await fs.readFile(inputFilePath, 'utf-8');
await fs.outputFile(
outputPath,
shouldCopyWithoutRender
? inputFileContents
: templater.renderString(inputFileContents, context),
);
}
}
}
ctx.logger.info(`Template result written to ${outputDir}`);
},
});
}