Merge pull request #2543 from mfrinnstrom/azure-scaffolder-prepare

Add Azure DevOps support to the scaffolder

plugins/scaffolder-backend/package.json

@@ -28,6 +28,7 @@
     "@octokit/rest": "^18.0.0",
     "@types/dockerode": "^2.5.32",
     "@types/express": "^4.17.6",
+    "azure-devops-node-api": "^10.1.1",
     "command-exists-promise": "^2.0.2",
     "compression": "^1.7.4",
     "cors": "^2.8.5",

plugins/scaffolder-backend/src/scaffolder/__mocks__/azure-devops-node-api/GitApi/index.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const mockGitApi = {
+  createRepository: jest.fn(),
+};
+
+export class GitApi {
+  constructor() {
+    return mockGitApi;
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const mocks = {
+  Clone: { clone: jest.fn() },
+  CheckoutOptions: jest.fn(() => {}),
+};
+jest.doMock('nodegit', () => mocks);
+
+import { AzurePreparer } from './azure';
+import {
+  TemplateEntityV1alpha1,
+  LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+
+describe('AzurePreparer', () => {
+  let mockEntity: TemplateEntityV1alpha1;
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Template',
+      metadata: {
+        annotations: {
+          [LOCATION_ANNOTATION]:
+            'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
+        },
+        name: 'graphql-starter',
+        title: 'GraphQL Service',
+        description:
+          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+        generation: 1,
+      },
+      spec: {
+        type: 'website',
+        templater: 'cookiecutter',
+        path: './template',
+        schema: {
+          $schema: 'http://json-schema.org/draft-07/schema#',
+          required: ['storePath', 'owner'],
+          properties: {
+            owner: {
+              type: 'string',
+              title: 'Owner',
+              description: 'Who is going to own this component',
+            },
+            storePath: {
+              type: 'string',
+              title: 'Store path',
+              description: 'GitHub store path in org/repo format',
+            },
+          },
+        },
+      },
+    };
+  });
+
+  it('calls the clone command with the correct arguments for a repository', async () => {
+    const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
+    await preparer.prepare(mockEntity);
+    expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
+      1,
+      'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
+      expect.any(String),
+      {},
+    );
+  });
+
+  it('calls the clone command with the correct arguments if an access token is provided for a repository', async () => {
+    const preparer = new AzurePreparer(
+      ConfigReader.fromConfigs([
+        {
+          context: '',
+          data: {
+            scaffolder: {
+              azure: {
+                api: {
+                  token: 'fake-token',
+                },
+              },
+            },
+          },
+        },
+      ]),
+    );
+    await preparer.prepare(mockEntity);
+    expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
+      1,
+      'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
+      expect.any(String),
+      {
+        fetchOpts: {
+          callbacks: {
+            credentials: expect.anything(),
+          },
+        },
+      },
+    );
+  });
+
+  it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
+    const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
+    delete mockEntity.spec.path;
+    await preparer.prepare(mockEntity);
+    expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
+      1,
+      'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
+      expect.any(String),
+      {},
+    );
+  });
+
+  it('return the temp directory with the path to the folder if it is specified', async () => {
+    const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
+    mockEntity.spec.path = './template/test/1/2/3';
+    const response = await preparer.prepare(mockEntity);
+
+    expect(response.split('\\').join('/')).toMatch(
+      /\/template\/test\/1\/2\/3$/,
+    );
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import os from 'os';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { parseLocationAnnotation } from '../helpers';
+import { InputError } from '@backstage/backend-common';
+import { PreparerBase } from './types';
+import GitUriParser from 'git-url-parse';
+import { Clone, Cred } from 'nodegit';
+import { Config } from '@backstage/config';
+
+export class AzurePreparer implements PreparerBase {
+  private readonly privateToken: string;
+
+  constructor(config: Config) {
+    this.privateToken =
+      config.getOptionalString('scaffolder.azure.api.token') ?? '';
+  }
+
+  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+    const { protocol, location } = parseLocationAnnotation(template);
+
+    if (protocol !== 'azure/api') {
+      throw new InputError(
+        `Wrong location protocol: ${protocol}, should be 'azure/api'`,
+      );
+    }
+    const templateId = template.metadata.name;
+
+    const url = new URL(location); // Need to extract filepath from search parameter
+    const parsedGitLocation = GitUriParser(location);
+    const repositoryCheckoutUrl = parsedGitLocation.toString('https');
+
+    const tempDir = await fs.promises.mkdtemp(
+      path.join(os.tmpdir(), templateId),
+    );
+
+    const templateDirectory = path.join(
+      `${path
+        .dirname(url.searchParams.get('path') || '')
+        .replace(/^\/+/g, '')}`, // Strip leading slash
+      template.spec.path ?? '.',
+    );
+
+    const options = this.privateToken
+      ? {
+          fetchOpts: {
+            callbacks: {
+              credentials: () =>
+                // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
+                Cred.userpassPlaintextNew('notempty', this.privateToken),
+            },
+          },
+        }
+      : {};
+
+    await Clone.clone(repositoryCheckoutUrl, tempDir, options);
+
+    return path.resolve(tempDir, templateDirectory);
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts

@@ -18,3 +18,4 @@ export * from './types';
 export * from './file';
 export * from './github';
 export * from './gitlab';
+export * from './azure';

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('nodegit');
+jest.mock('azure-devops-node-api/GitApi');
+jest.mock('azure-devops-node-api/interfaces/GitInterfaces');
+
+import { AzurePublisher } from './azure';
+import { GitApi } from 'azure-devops-node-api/GitApi';
+import * as NodeGit from 'nodegit';
+
+const { mockGitApi } = require('azure-devops-node-api/GitApi') as {
+  mockGitApi: {
+    createRepository: jest.MockedFunction<GitApi['createRepository']>;
+  };
+};
+
+const {
+  Repository,
+  mockRepo,
+  mockIndex,
+  Signature,
+  Remote,
+  mockRemote,
+  Cred,
+} = require('nodegit') as {
+  Repository: jest.Mocked<{ init: any }>;
+  Signature: jest.Mocked<{ now: any }>;
+  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
+  Remote: jest.Mocked<{ create: any }>;
+
+  mockIndex: jest.Mocked<NodeGit.Index>;
+  mockRepo: jest.Mocked<NodeGit.Repository>;
+  mockRemote: jest.Mocked<NodeGit.Remote>;
+};
+
+describe('Azure Publisher', () => {
+  const publisher = new AzurePublisher(new GitApi('', []), 'fake-token');
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('publish: createRemoteInAzure', () => {
+    it('should use azure-devops-node-api to create a repo in the given project', async () => {
+      mockGitApi.createRepository.mockResolvedValue({
+        remoteUrl: 'mockclone',
+      } as { remoteUrl: string });
+
+      await publisher.publish({
+        values: {
+          storePath: 'project/repo',
+          owner: 'bob',
+        },
+        directory: '/tmp/test',
+      });
+
+      expect(mockGitApi.createRepository).toHaveBeenCalledWith(
+        {
+          name: 'repo',
+        },
+        'project',
+      );
+    });
+  });
+
+  describe('publish: createGitDirectory', () => {
+    const values = {
+      isOrg: true,
+      storePath: 'blam/test',
+      owner: 'lols',
+    };
+
+    const mockDir = '/tmp/test/dir';
+
+    mockGitApi.createRepository.mockResolvedValue({
+      remoteUrl: 'mockclone',
+    } as { remoteUrl: string });
+
+    it('should call init on the repo with the directory', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
+    });
+
+    it('should call refresh index on the index and write the new files', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(mockRepo.refreshIndex).toHaveBeenCalled();
+    });
+
+    it('should call add all files and write', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(mockIndex.addAll).toHaveBeenCalled();
+      expect(mockIndex.write).toHaveBeenCalled();
+      expect(mockIndex.writeTree).toHaveBeenCalled();
+    });
+
+    it('should create a commit with on head with the right name and commiter', async () => {
+      const mockSignature = { mockSignature: 'bloblly' };
+      Signature.now.mockReturnValue(mockSignature);
+
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Signature.now).toHaveBeenCalledTimes(2);
+      expect(Signature.now).toHaveBeenCalledWith(
+        'Scaffolder',
+        'scaffolder@backstage.io',
+      );
+
+      expect(mockRepo.createCommit).toHaveBeenCalledWith(
+        'HEAD',
+        mockSignature,
+        mockSignature,
+        'initial commit',
+        'mockoid',
+        [],
+      );
+    });
+
+    it('creates a remote with the repo and remote', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Remote.create).toHaveBeenCalledWith(
+        mockRepo,
+        'origin',
+        'mockclone',
+      );
+    });
+
+    it('shoud push to the remote repo', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      const [remotes, { callbacks }] = mockRemote.push.mock
+        .calls[0] as NodeGit.PushOptions[];
+
+      expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
+
+      callbacks?.credentials?.();
+
+      expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
+        'notempty',
+        'fake-token',
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PublisherBase } from './types';
+import { GitApi } from 'azure-devops-node-api/GitApi';
+import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces';
+
+import { JsonValue } from '@backstage/config';
+import { RequiredTemplateValues } from '../templater';
+import { Repository, Remote, Signature, Cred } from 'nodegit';
+
+export class AzurePublisher implements PublisherBase {
+  private readonly client: GitApi;
+  private readonly token: string;
+
+  constructor(client: GitApi, token: string) {
+    this.client = client;
+    this.token = token;
+  }
+
+  async publish({
+    values,
+    directory,
+  }: {
+    values: RequiredTemplateValues & Record<string, JsonValue>;
+    directory: string;
+  }): Promise<{ remoteUrl: string }> {
+    const remoteUrl = await this.createRemote(values);
+    await this.pushToRemote(directory, remoteUrl);
+
+    return { remoteUrl };
+  }
+
+  private async createRemote(
+    values: RequiredTemplateValues & Record<string, JsonValue>,
+  ) {
+    const [project, name] = values.storePath.split('/');
+
+    const createOptions: GitRepositoryCreateOptions = { name };
+    const repo = await this.client.createRepository(createOptions, project);
+
+    return repo.remoteUrl || '';
+  }
+
+  private async pushToRemote(directory: string, remote: string): Promise<void> {
+    const repo = await Repository.init(directory, 0);
+    const index = await repo.refreshIndex();
+    await index.addAll();
+    await index.write();
+    const oid = await index.writeTree();
+    await repo.createCommit(
+      'HEAD',
+      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
+      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
+      'initial commit',
+      oid,
+      [],
+    );
+
+    const remoteRepo = await Remote.create(repo, 'origin', remote);
+
+    await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
+      callbacks: {
+        // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
+        credentials: () => Cred.userpassPlaintextNew('notempty', this.token),
+      },
+    });
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts

@@ -16,4 +16,5 @@
 export * from './publishers';
 export * from './github';
 export * from './gitlab';
+export * from './azure';
 export * from './types';

plugins/scaffolder-backend/src/scaffolder/stages/types.ts

@@ -13,4 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export type RemoteProtocol = 'file' | 'github' | 'gitlab' | 'gitlab/api';
+export type RemoteProtocol =
+  | 'file'
+  | 'github'
+  | 'gitlab'
+  | 'gitlab/api'
+  | 'azure/api';