Merge pull request #3432 from backstage/orkohunter/techdocs-publish-to-cloud-storage
This commit is contained in:
Himanshu Mishra
@@ -18,30 +18,13 @@ yarn start
|
||||
|
||||
## What techdocs-backend does
|
||||
|
||||
This plugin is the backend part of the techdocs plugin. It provides building and serving of your docs without having to use another service and hosting provider. To use it set your techdocs storageUrl in your `app-config.yml` to `http://localhost:7000/api/techdocs/static/docs`.
|
||||
This plugin is the backend part of the techdocs plugin. It provides serving and building of documentation for any entity.
|
||||
To configure various storage providers and building options, see http://backstage.io/docs/features/techdocs/configuration
|
||||
|
||||
```yaml
|
||||
techdocs:
|
||||
storageUrl: http://localhost:7000/api/techdocs/static/docs
|
||||
```
|
||||
|
||||
## Extending techdocs-backend
|
||||
|
||||
Currently the build process of techdocs-backend is split up in these three stages.
|
||||
|
||||
- Preparers
|
||||
- Generators
|
||||
- Publishers
|
||||
|
||||
Preparers read your entity data and creates a working directory with your documentation source code. For example if you have set your `backstage.io/techdocs-ref` to `github:https://github.com/backstage/backstage.git` it will clone that repository to a temp folder and pass that on to the generator.
|
||||
|
||||
Generators takes the prepared source and runs the `techdocs-container` on it. It then passes on the output folder of that build to the publisher.
|
||||
|
||||
Publishers gets a folder path from the generator and publish it to your storage solution. Currently the only built in storage solution is a folder called `static/docs` inside the techdocs-backend plugin.
|
||||
|
||||
Any of these can be extended. If we want to publish to a external static file server using rsync for example that can be done by creating a rsync publisher. _(Keep in mind that if you want techdocs-backend to initiate a build this would also require techdocs-backend to act as a proxy, which is not yet implemented.)_
|
||||
The techdocs-backend re-exports the [techdocs-common](https://github.com/backstage/backstage/tree/master/packages/techdocs-common) package which has the features to prepare, generate and publish docs.
|
||||
The Publishers are also used to fetch the static documentation files and render them in TechDocs.
|
||||
|
||||
## Links
|
||||
|
||||
- [Frontend part of the plugin](https://github.com/backstage/backstage/tree/master/plugins/techdocs)
|
||||
- [The Backstage homepage](https://backstage.io)
|
||||
- [Backstage homepage](https://backstage.io)
|
||||
|
||||
@@ -33,19 +33,14 @@
|
||||
"@backstage/backend-common": "^0.4.0",
|
||||
"@backstage/catalog-model": "^0.5.0",
|
||||
"@backstage/config": "^0.1.2",
|
||||
"@backstage/techdocs-common": "^0.1.1",
|
||||
"@types/dockerode": "^3.2.1",
|
||||
"@types/express": "^4.17.6",
|
||||
"command-exists-promise": "^2.0.2",
|
||||
"cross-fetch": "^3.0.6",
|
||||
"dockerode": "^3.2.1",
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^3.0.3",
|
||||
"fs-extra": "^9.0.1",
|
||||
"git-url-parse": "^11.4.0",
|
||||
"js-yaml": "^3.14.0",
|
||||
"knex": "^0.21.6",
|
||||
"mock-fs": "^4.13.0",
|
||||
"nodegit": "^0.27.0",
|
||||
"winston": "^3.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
+11
-3
@@ -13,7 +13,15 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { BuildMetadataStorage } from './BuildMetadataStorage';
|
||||
|
||||
export * from './generate';
|
||||
export * from './prepare';
|
||||
export * from './publish';
|
||||
describe('BuildMetadataStorage', () => {
|
||||
it('should return build timestamp', () => {
|
||||
const newMetadataStorage = new BuildMetadataStorage('123abc');
|
||||
newMetadataStorage.storeBuildTimestamp();
|
||||
|
||||
const timestamp = newMetadataStorage.getTimestamp();
|
||||
|
||||
expect(timestamp).toBeLessThanOrEqual(Date.now());
|
||||
});
|
||||
});
|
||||
+5
@@ -20,6 +20,11 @@ type buildInfo = {
|
||||
|
||||
const builds = {} as buildInfo;
|
||||
|
||||
/**
|
||||
* Store timestamps of the most recent TechDocs build of each Entity. This is
|
||||
* used to invalidate cache if the latest commit in the documentation source
|
||||
* repository is later than the timestamp.
|
||||
*/
|
||||
export class BuildMetadataStorage {
|
||||
public entityUid: string;
|
||||
private builds: buildInfo;
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import Docker from 'dockerode';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import {
|
||||
PreparerBuilder,
|
||||
PublisherBase,
|
||||
GeneratorBuilder,
|
||||
PreparerBase,
|
||||
GeneratorBase,
|
||||
getLocationForEntity,
|
||||
getLastCommitTimestamp,
|
||||
} from '@backstage/techdocs-common';
|
||||
import { BuildMetadataStorage } from '.';
|
||||
|
||||
const getEntityId = (entity: Entity) => {
|
||||
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
|
||||
entity.metadata.name
|
||||
}`;
|
||||
};
|
||||
|
||||
type DocsBuilderArguments = {
|
||||
preparers: PreparerBuilder;
|
||||
generators: GeneratorBuilder;
|
||||
publisher: PublisherBase;
|
||||
entity: Entity;
|
||||
logger: Logger;
|
||||
dockerClient: Docker;
|
||||
};
|
||||
|
||||
export class DocsBuilder {
|
||||
private preparer: PreparerBase;
|
||||
private generator: GeneratorBase;
|
||||
private publisher: PublisherBase;
|
||||
private entity: Entity;
|
||||
private logger: Logger;
|
||||
private dockerClient: Docker;
|
||||
|
||||
constructor({
|
||||
preparers,
|
||||
generators,
|
||||
publisher,
|
||||
entity,
|
||||
logger,
|
||||
dockerClient,
|
||||
}: DocsBuilderArguments) {
|
||||
this.preparer = preparers.get(entity);
|
||||
this.generator = generators.get(entity);
|
||||
this.publisher = publisher;
|
||||
this.entity = entity;
|
||||
this.logger = logger;
|
||||
this.dockerClient = dockerClient;
|
||||
}
|
||||
|
||||
public async build() {
|
||||
this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
|
||||
const preparedDir = await this.preparer.prepare(this.entity);
|
||||
|
||||
const parsedLocationAnnotation = getLocationForEntity(this.entity);
|
||||
|
||||
this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
|
||||
const { resultDir } = await this.generator.run({
|
||||
directory: preparedDir,
|
||||
dockerClient: this.dockerClient,
|
||||
parsedLocationAnnotation,
|
||||
});
|
||||
|
||||
this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);
|
||||
await this.publisher.publish({
|
||||
entity: this.entity,
|
||||
directory: resultDir,
|
||||
});
|
||||
|
||||
if (!this.entity.metadata.uid) {
|
||||
throw new Error(
|
||||
'Trying to build documentation for entity not in service catalog',
|
||||
);
|
||||
}
|
||||
|
||||
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
|
||||
}
|
||||
|
||||
public async docsUpToDate() {
|
||||
if (!this.entity.metadata.uid) {
|
||||
throw new Error(
|
||||
'Trying to build documentation for entity not in service catalog',
|
||||
);
|
||||
}
|
||||
|
||||
const buildMetadataStorage = new BuildMetadataStorage(
|
||||
this.entity.metadata.uid,
|
||||
);
|
||||
const { type, target } = getLocationForEntity(this.entity);
|
||||
|
||||
// Unless docs are stored locally
|
||||
const nonAgeCheckTypes = ['dir', 'file', 'url'];
|
||||
if (!nonAgeCheckTypes.includes(type)) {
|
||||
const lastCommit = await getLastCommitTimestamp(target, this.logger);
|
||||
const storageTimeStamp = buildMetadataStorage.getTimestamp();
|
||||
|
||||
// Check if documentation source is newer than what we have
|
||||
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
|
||||
this.logger.debug(
|
||||
`Docs for entity ${getEntityId(this.entity)} is up to date.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Docs for entity ${getEntityId(this.entity)} was outdated.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+1
@@ -14,3 +14,4 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
export * from './BuildMetadataStorage';
|
||||
export * from './builder';
|
||||
@@ -1,261 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fetch from 'cross-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
|
||||
import {
|
||||
getAzureHostToken,
|
||||
getGitHost,
|
||||
getGithubHostToken,
|
||||
getGitlabHostToken,
|
||||
getGitRepoType,
|
||||
} from './git-auth';
|
||||
|
||||
interface IGitlabBranch {
|
||||
name: string;
|
||||
merged: boolean;
|
||||
protected: boolean;
|
||||
default: boolean;
|
||||
developers_can_push: boolean;
|
||||
developers_can_merge: boolean;
|
||||
can_push: boolean;
|
||||
web_url: string;
|
||||
commit: {
|
||||
author_email: string;
|
||||
author_name: string;
|
||||
authored_date: string;
|
||||
committed_date: string;
|
||||
committer_email: string;
|
||||
committer_name: string;
|
||||
id: string;
|
||||
short_id: string;
|
||||
title: string;
|
||||
message: string;
|
||||
parent_ids: string[];
|
||||
};
|
||||
}
|
||||
|
||||
function getGithubApiUrl(config: Config, url: string): URL {
|
||||
const { protocol, owner, name } = parseGitUrl(url);
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.github') ?? [];
|
||||
|
||||
// TODO: Maybe we need to filter by host in the array, not sure about GHE
|
||||
const targetProviderConfig = providerConfigs[0];
|
||||
|
||||
const apiBaseUrl =
|
||||
targetProviderConfig?.getOptionalString('integrations.github.apiBaseUrl') ??
|
||||
'api.github.com';
|
||||
const apiRepos = 'repos';
|
||||
|
||||
return new URL(`${protocol}://${apiBaseUrl}/${apiRepos}/${owner}/${name}`);
|
||||
}
|
||||
|
||||
function getGitlabApiUrl(url: string): URL {
|
||||
const { protocol, resource, full_name: fullName } = parseGitUrl(url);
|
||||
const apiProjectsBasePath = 'api/v4/projects';
|
||||
const project = encodeURIComponent(fullName);
|
||||
const branches = 'repository/branches';
|
||||
|
||||
return new URL(
|
||||
`${protocol}://${resource}/${apiProjectsBasePath}/${project}/${branches}`,
|
||||
);
|
||||
}
|
||||
|
||||
function getAzureApiUrl(url: string): URL {
|
||||
const { protocol, resource, organization, owner, name } = parseGitUrl(url);
|
||||
const apiRepoPath = '_apis/git/repositories';
|
||||
const apiVersion = 'api-version=6.0';
|
||||
|
||||
return new URL(
|
||||
`${protocol}://${resource}/${organization}/${owner}/${apiRepoPath}/${name}?${apiVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
function getGithubRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
const token = getGithubHostToken(config, host);
|
||||
|
||||
if (token) {
|
||||
headers.Authorization = `token ${token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
function getGitlabRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
'PRIVATE-TOKEN': '',
|
||||
};
|
||||
|
||||
const token = getGitlabHostToken(config, host);
|
||||
if (token) {
|
||||
headers['PRIVATE-TOKEN'] = token;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
function getAzureRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
const token = getAzureHostToken(config, host);
|
||||
|
||||
if (token !== '') {
|
||||
headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString(
|
||||
'base64',
|
||||
)}`;
|
||||
}
|
||||
|
||||
const requestOptions: RequestInit = {
|
||||
headers,
|
||||
};
|
||||
|
||||
return requestOptions;
|
||||
}
|
||||
|
||||
async function getGithubDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getGithubApiUrl(config, repositoryUrl).toString();
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const options = getGithubRequestOptions(config, host);
|
||||
|
||||
try {
|
||||
const raw = await fetch(path, options);
|
||||
|
||||
if (!raw.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const { default_branch: branch } = await raw.json();
|
||||
|
||||
if (!branch) {
|
||||
throw new Error('Not found github default branch');
|
||||
}
|
||||
|
||||
return branch;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get github default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function getGitlabDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getGitlabApiUrl(repositoryUrl).toString();
|
||||
|
||||
const gitlabHost = getGitHost(repositoryUrl);
|
||||
const options = getGitlabRequestOptions(config, gitlabHost);
|
||||
|
||||
try {
|
||||
const raw = await fetch(path, options);
|
||||
|
||||
if (!raw.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await raw.json();
|
||||
const { name } = (result || []).find(
|
||||
(branch: IGitlabBranch) => branch.default === true,
|
||||
);
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Not found gitlab default branch');
|
||||
}
|
||||
|
||||
return name;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get gitlab default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function getAzureDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getAzureApiUrl(repositoryUrl).toString();
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const options = getAzureRequestOptions(config, host);
|
||||
|
||||
try {
|
||||
const urlResponse = await fetch(path, options);
|
||||
if (!urlResponse.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${urlResponse.status} ${urlResponse.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
const urlResult = await urlResponse.json();
|
||||
|
||||
const idResponse = await fetch(urlResult.url, options);
|
||||
if (!idResponse.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${idResponse.status} ${idResponse.statusText}. Make sure you have permission to repository: ${urlResult.repository.url}`,
|
||||
);
|
||||
}
|
||||
const idResult = await idResponse.json();
|
||||
const name = idResult.defaultBranch;
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Not found Azure DevOps default branch');
|
||||
}
|
||||
|
||||
return name;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get Azure DevOps default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const getDefaultBranch = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string> => {
|
||||
// TODO(Rugvip): Config should not be loaded here, pass it in instead
|
||||
const config = await loadBackendConfig({
|
||||
logger: getRootLogger(),
|
||||
argv: process.argv,
|
||||
});
|
||||
const type = getGitRepoType(repositoryUrl);
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'github':
|
||||
return await getGithubDefaultBranch(repositoryUrl, config);
|
||||
case 'gitlab':
|
||||
return await getGitlabDefaultBranch(repositoryUrl, config);
|
||||
case 'azure/api':
|
||||
return await getAzureDefaultBranch(repositoryUrl, config);
|
||||
|
||||
default:
|
||||
throw new Error('Failed to get repository type');
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -1,120 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
|
||||
|
||||
export function getGitHost(url: string): string {
|
||||
const { resource } = parseGitUrl(url);
|
||||
return resource;
|
||||
}
|
||||
|
||||
export function getGitRepoType(url: string): string {
|
||||
const typeMapping = [
|
||||
{ url: /github*/g, type: 'github' },
|
||||
{ url: /gitlab*/g, type: 'gitlab' },
|
||||
{ url: /azure*/g, type: 'azure/api' },
|
||||
];
|
||||
|
||||
const type = typeMapping.filter(item => item.url.test(url))[0]?.type;
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
export function getGithubHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.github') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.github.privateToken') ??
|
||||
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
|
||||
process.env.GITHUB_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export function getGitlabHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.gitlab') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.gitlab.privateToken') ??
|
||||
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
|
||||
process.env.GITLAB_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export function getAzureHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.azure') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.azureApi.privateToken') ??
|
||||
process.env.AZURE_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export const getTokenForGitRepo = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string | undefined> => {
|
||||
// TODO(Rugvip): Config should not be loaded here, pass it in instead
|
||||
const config = await loadBackendConfig({
|
||||
logger: getRootLogger(),
|
||||
argv: process.argv,
|
||||
});
|
||||
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const type = getGitRepoType(repositoryUrl);
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'github':
|
||||
return getGithubHostToken(config, host);
|
||||
case 'gitlab':
|
||||
return getGitlabHostToken(config, host);
|
||||
case 'azure/api':
|
||||
return getAzureHostToken(config, host);
|
||||
|
||||
default:
|
||||
throw new Error('Failed to get repository type');
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -1,70 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Readable } from 'stream';
|
||||
import { getDocFilesFromRepository } from './helpers';
|
||||
import { UrlReader, ReadTreeResponse } from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
|
||||
describe('getDocFilesFromRepository', () => {
|
||||
it('should read a remote directory using UrlReader.readTree', async () => {
|
||||
class MockUrlReader implements UrlReader {
|
||||
async read() {
|
||||
return Buffer.from('mock');
|
||||
}
|
||||
|
||||
async readTree(): Promise<ReadTreeResponse> {
|
||||
return {
|
||||
dir: async () => {
|
||||
return '/tmp/testfolder';
|
||||
},
|
||||
files: async () => {
|
||||
return [];
|
||||
},
|
||||
archive: async () => {
|
||||
return Readable.from('');
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const mockEntity: Entity = {
|
||||
metadata: {
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
'backstage.io/techdocs-ref':
|
||||
'url:https://github.com/backstage/backstage/blob/master/subfolder/',
|
||||
},
|
||||
name: 'mytestcomponent',
|
||||
description: 'A component for testing',
|
||||
},
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
spec: {
|
||||
type: 'documentation',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'testuser',
|
||||
},
|
||||
};
|
||||
|
||||
const output = await getDocFilesFromRepository(
|
||||
new MockUrlReader(),
|
||||
mockEntity,
|
||||
);
|
||||
|
||||
expect(output).toBe('/tmp/testfolder');
|
||||
});
|
||||
});
|
||||
@@ -1,195 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import NodeGit, { Clone, Repository } from 'nodegit';
|
||||
import fs from 'fs-extra';
|
||||
import { getDefaultBranch } from './default-branch';
|
||||
import { getGitRepoType, getTokenForGitRepo } from './git-auth';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { InputError, UrlReader } from '@backstage/backend-common';
|
||||
import { RemoteProtocol } from './techdocs/stages/prepare/types';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
// Enables core.longpaths on windows to prevent crashing when checking out repos with long foldernames and/or deep nesting
|
||||
// @ts-ignore
|
||||
NodeGit.Libgit2.opts(28, 1);
|
||||
|
||||
export type ParsedLocationAnnotation = {
|
||||
type: RemoteProtocol;
|
||||
target: string;
|
||||
};
|
||||
|
||||
export const parseReferenceAnnotation = (
|
||||
annotationName: string,
|
||||
entity: Entity,
|
||||
): ParsedLocationAnnotation => {
|
||||
const annotation = entity.metadata.annotations?.[annotationName];
|
||||
|
||||
if (!annotation) {
|
||||
throw new InputError(
|
||||
`No location annotation provided in entity: ${entity.metadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
// split on the first colon for the protocol and the rest after the first split
|
||||
// is the location.
|
||||
const [type, target] = annotation.split(/:(.+)/) as [
|
||||
RemoteProtocol?,
|
||||
string?,
|
||||
];
|
||||
|
||||
if (!type || !target) {
|
||||
throw new InputError(
|
||||
`Failure to parse either protocol or location for entity: ${entity.metadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
type,
|
||||
target,
|
||||
};
|
||||
};
|
||||
|
||||
export const getLocationForEntity = (
|
||||
entity: Entity,
|
||||
): ParsedLocationAnnotation => {
|
||||
const { type, target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
switch (type) {
|
||||
case 'github':
|
||||
case 'gitlab':
|
||||
case 'azure/api':
|
||||
case 'url':
|
||||
return { type, target };
|
||||
case 'dir':
|
||||
if (path.isAbsolute(target)) return { type, target };
|
||||
|
||||
return parseReferenceAnnotation(
|
||||
'backstage.io/managed-by-location',
|
||||
entity,
|
||||
);
|
||||
default:
|
||||
throw new Error(`Invalid reference annotation ${type}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const getGitRepositoryTempFolder = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string> => {
|
||||
const parsedGitLocation = parseGitUrl(repositoryUrl);
|
||||
// removes .git from git location path
|
||||
parsedGitLocation.git_suffix = false;
|
||||
|
||||
if (!parsedGitLocation.ref) {
|
||||
parsedGitLocation.ref = await getDefaultBranch(
|
||||
parsedGitLocation.toString('https'),
|
||||
);
|
||||
}
|
||||
|
||||
return path.join(
|
||||
// fs.realpathSync fixes a problem with macOS returning a path that is a symlink
|
||||
fs.realpathSync(os.tmpdir()),
|
||||
'backstage-repo',
|
||||
parsedGitLocation.source,
|
||||
parsedGitLocation.owner,
|
||||
parsedGitLocation.name,
|
||||
parsedGitLocation.ref,
|
||||
);
|
||||
};
|
||||
|
||||
export const checkoutGitRepository = async (
|
||||
repoUrl: string,
|
||||
logger: Logger,
|
||||
): Promise<string> => {
|
||||
const parsedGitLocation = parseGitUrl(repoUrl);
|
||||
const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl);
|
||||
const token = await getTokenForGitRepo(repoUrl);
|
||||
|
||||
if (fs.existsSync(repositoryTmpPath)) {
|
||||
try {
|
||||
const repository = await Repository.open(repositoryTmpPath);
|
||||
const currentBranchName = (
|
||||
await repository.getCurrentBranch()
|
||||
).shorthand();
|
||||
await repository.fetch('origin');
|
||||
await repository.mergeBranches(
|
||||
currentBranchName,
|
||||
`origin/${currentBranchName}`,
|
||||
);
|
||||
return repositoryTmpPath;
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Found error "${e.message}" in cached repository "${repoUrl}" when getting latest changes. Removing cached repository.`,
|
||||
);
|
||||
fs.removeSync(repositoryTmpPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (token) {
|
||||
const type = getGitRepoType(repoUrl);
|
||||
switch (type) {
|
||||
case 'gitlab':
|
||||
// Personal Access Token
|
||||
parsedGitLocation.token = `dummyUsername:${token}`;
|
||||
parsedGitLocation.git_suffix = true;
|
||||
break;
|
||||
case 'github':
|
||||
parsedGitLocation.token = `${token}:x-oauth-basic`;
|
||||
break;
|
||||
default:
|
||||
parsedGitLocation.token = `:${token}`;
|
||||
}
|
||||
}
|
||||
|
||||
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
|
||||
|
||||
fs.mkdirSync(repositoryTmpPath, { recursive: true });
|
||||
await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath);
|
||||
|
||||
return repositoryTmpPath;
|
||||
};
|
||||
|
||||
export const getLastCommitTimestamp = async (
|
||||
repositoryUrl: string,
|
||||
logger: Logger,
|
||||
): Promise<number> => {
|
||||
const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger);
|
||||
|
||||
const repository = await Repository.open(repositoryLocation);
|
||||
const commit = await repository.getReferenceCommit('HEAD');
|
||||
|
||||
return commit.date().getTime();
|
||||
};
|
||||
|
||||
export const getDocFilesFromRepository = async (
|
||||
reader: UrlReader,
|
||||
entity: Entity,
|
||||
): Promise<any> => {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
const response = await reader.readTree(target);
|
||||
|
||||
return await response.dir();
|
||||
};
|
||||
@@ -15,4 +15,4 @@
|
||||
*/
|
||||
|
||||
export * from './service/router';
|
||||
export * from './techdocs';
|
||||
export * from '@backstage/techdocs-common';
|
||||
|
||||
+10
-16
@@ -13,20 +13,14 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { getEntityNameFromUrlPath } from './helpers';
|
||||
|
||||
/**
|
||||
* Publisher is in charge of taking a folder created by
|
||||
* the builder, and pushing it to storage
|
||||
*/
|
||||
export type PublisherBase = {
|
||||
/**
|
||||
*
|
||||
* @param opts object containing the entity from the service
|
||||
* catalog, and the directory that has been generated
|
||||
*/
|
||||
publish(opts: {
|
||||
entity: Entity;
|
||||
directory: string;
|
||||
}): Promise<{ remoteUrl: string }> | { remoteUrl: string };
|
||||
};
|
||||
describe('getEntityNameFromUrlPath', () => {
|
||||
it('should parse correctly', () => {
|
||||
const path = 'default/Component/documented-component';
|
||||
const parsedEntity = getEntityNameFromUrlPath(path);
|
||||
expect(parsedEntity).toHaveProperty('namespace', 'default');
|
||||
expect(parsedEntity).toHaveProperty('kind', 'Component');
|
||||
expect(parsedEntity).toHaveProperty('name', 'documented-component');
|
||||
});
|
||||
});
|
||||
@@ -13,126 +13,18 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import Docker from 'dockerode';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import {
|
||||
PreparerBuilder,
|
||||
PublisherBase,
|
||||
GeneratorBuilder,
|
||||
PreparerBase,
|
||||
GeneratorBase,
|
||||
} from '../techdocs';
|
||||
import { BuildMetadataStorage } from '../storage';
|
||||
import { getLocationForEntity, getLastCommitTimestamp } from '../helpers';
|
||||
import { EntityName } from '@backstage/catalog-model';
|
||||
/**
|
||||
* Using the path of the TechDocs page URL, return a structured EntityName type object with namespace,
|
||||
* kind and name of the Entity.
|
||||
* @param {string} path Example: default/Component/documented-component
|
||||
*/
|
||||
export const getEntityNameFromUrlPath = (path: string): EntityName => {
|
||||
const [namespace, kind, name] = path.split('/');
|
||||
|
||||
const getEntityId = (entity: Entity) => {
|
||||
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
|
||||
entity.metadata.name
|
||||
}`;
|
||||
return {
|
||||
namespace,
|
||||
kind,
|
||||
name,
|
||||
};
|
||||
};
|
||||
|
||||
type DocsBuilderArguments = {
|
||||
preparers: PreparerBuilder;
|
||||
generators: GeneratorBuilder;
|
||||
publisher: PublisherBase;
|
||||
entity: Entity;
|
||||
logger: Logger;
|
||||
dockerClient: Docker;
|
||||
};
|
||||
|
||||
export class DocsBuilder {
|
||||
private preparer: PreparerBase;
|
||||
private generator: GeneratorBase;
|
||||
private publisher: PublisherBase;
|
||||
private entity: Entity;
|
||||
private logger: Logger;
|
||||
private dockerClient: Docker;
|
||||
|
||||
constructor({
|
||||
preparers,
|
||||
generators,
|
||||
publisher,
|
||||
entity,
|
||||
logger,
|
||||
dockerClient,
|
||||
}: DocsBuilderArguments) {
|
||||
this.preparer = preparers.get(entity);
|
||||
this.generator = generators.get(entity);
|
||||
this.publisher = publisher;
|
||||
this.entity = entity;
|
||||
this.logger = logger;
|
||||
this.dockerClient = dockerClient;
|
||||
}
|
||||
|
||||
public async build() {
|
||||
this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
|
||||
const preparedDir = await this.preparer.prepare(this.entity);
|
||||
|
||||
const parsedLocationAnnotation = getLocationForEntity(this.entity);
|
||||
|
||||
this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
|
||||
const { resultDir } = await this.generator.run({
|
||||
directory: preparedDir,
|
||||
dockerClient: this.dockerClient,
|
||||
parsedLocationAnnotation,
|
||||
});
|
||||
|
||||
this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);
|
||||
await this.publisher.publish({
|
||||
entity: this.entity,
|
||||
directory: resultDir,
|
||||
});
|
||||
|
||||
if (!this.entity.metadata.uid) {
|
||||
throw new Error(
|
||||
'Trying to build documentation for entity not in service catalog',
|
||||
);
|
||||
}
|
||||
|
||||
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
|
||||
}
|
||||
|
||||
public async docsUpToDate() {
|
||||
if (!this.entity.metadata.uid) {
|
||||
throw new Error(
|
||||
'Trying to build documentation for entity not in service catalog',
|
||||
);
|
||||
}
|
||||
|
||||
const buildMetadataStorage = new BuildMetadataStorage(
|
||||
this.entity.metadata.uid,
|
||||
);
|
||||
const { type, target } = getLocationForEntity(this.entity);
|
||||
|
||||
// Unless docs are stored locally
|
||||
const nonAgeCheckTypes = ['dir', 'file', 'url'];
|
||||
if (!nonAgeCheckTypes.includes(type)) {
|
||||
const lastCommit = await getLastCommitTimestamp(target, this.logger);
|
||||
const storageTimeStamp = buildMetadataStorage.getTimestamp();
|
||||
|
||||
// Check if documentation source is newer than what we have
|
||||
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
|
||||
this.logger.debug(
|
||||
`Docs for entity ${getEntityId(this.entity)} is up to date.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Better caching for URL.
|
||||
if (type === 'url') {
|
||||
const builtAt = buildMetadataStorage.getTimestamp();
|
||||
const now = Date.now();
|
||||
|
||||
if (builtAt > now - 1800000) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`Docs for entity ${getEntityId(this.entity)} was outdated.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,15 +24,12 @@ import {
|
||||
GeneratorBuilder,
|
||||
PreparerBuilder,
|
||||
PublisherBase,
|
||||
LocalPublish,
|
||||
} from '../techdocs';
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
resolvePackagePath,
|
||||
} from '@backstage/backend-common';
|
||||
getLocationForEntity,
|
||||
} from '@backstage/techdocs-common';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { DocsBuilder } from './helpers';
|
||||
import { getLocationForEntity } from '../helpers';
|
||||
import { getEntityNameFromUrlPath } from './helpers';
|
||||
import { DocsBuilder } from '../DocsBuilder';
|
||||
|
||||
type RouterOptions = {
|
||||
preparers: PreparerBuilder;
|
||||
@@ -45,11 +42,6 @@ type RouterOptions = {
|
||||
dockerClient: Docker;
|
||||
};
|
||||
|
||||
const staticDocsDir = resolvePackagePath(
|
||||
'@backstage/plugin-techdocs-backend',
|
||||
'static/docs',
|
||||
);
|
||||
|
||||
export async function createRouter({
|
||||
preparers,
|
||||
generators,
|
||||
@@ -62,24 +54,18 @@ export async function createRouter({
|
||||
const router = Router();
|
||||
|
||||
router.get('/metadata/techdocs/*', async (req, res) => {
|
||||
let storageUrl = config.getString('techdocs.storageUrl');
|
||||
if (publisher instanceof LocalPublish) {
|
||||
storageUrl = new URL(
|
||||
new URL(storageUrl).pathname,
|
||||
await discovery.getBaseUrl('techdocs'),
|
||||
).toString();
|
||||
}
|
||||
// path is `:namespace/:kind:/:name`
|
||||
const { '0': path } = req.params;
|
||||
const entityName = getEntityNameFromUrlPath(path);
|
||||
|
||||
const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`;
|
||||
|
||||
try {
|
||||
const techdocsMetadata = await (await fetch(metadataURL)).json();
|
||||
res.send(techdocsMetadata);
|
||||
} catch (err) {
|
||||
logger.info(`Unable to get metadata for ${path} with error ${err}`);
|
||||
throw new Error(`Unable to get metadata for ${path} with error ${err}`);
|
||||
}
|
||||
publisher
|
||||
.fetchTechDocsMetadata(entityName)
|
||||
.then(techdocsMetadataJson => {
|
||||
res.send(techdocsMetadataJson);
|
||||
})
|
||||
.catch(reason => {
|
||||
res.status(500).send(`Unable to get Metadata. Reason: ${reason}`);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => {
|
||||
@@ -107,9 +93,8 @@ export async function createRouter({
|
||||
});
|
||||
|
||||
router.get('/docs/:namespace/:kind/:name/*', async (req, res) => {
|
||||
const storageUrl = config.getString('techdocs.storageUrl');
|
||||
|
||||
const { kind, namespace, name } = req.params;
|
||||
const storageUrl = config.getString('techdocs.storageUrl');
|
||||
|
||||
const catalogUrl = await discovery.getBaseUrl('catalog');
|
||||
const triple = [kind, namespace, name].map(encodeURIComponent).join('/');
|
||||
@@ -124,25 +109,80 @@ export async function createRouter({
|
||||
|
||||
const entity: Entity = await catalogRes.json();
|
||||
|
||||
const docsBuilder = new DocsBuilder({
|
||||
preparers,
|
||||
generators,
|
||||
publisher,
|
||||
dockerClient,
|
||||
logger,
|
||||
entity,
|
||||
});
|
||||
let publisherType = '';
|
||||
try {
|
||||
publisherType = config.getString('techdocs.publisher.type');
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
'Unable to get techdocs.publisher.type in your app config. Set it to either ' +
|
||||
"'local', 'googleGcs' or other support storage providers. Read more here " +
|
||||
'https://backstage.io/docs/features/techdocs/architecture',
|
||||
);
|
||||
}
|
||||
|
||||
if (!(await docsBuilder.docsUpToDate())) {
|
||||
await docsBuilder.build();
|
||||
// techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local'
|
||||
// If set to 'external', it will only try to fetch and assume that an external process (e.g. CI/CD pipeline
|
||||
// of the repository) is responsible for building and publishing documentation to the storage provider.
|
||||
if (config.getString('techdocs.builder') === 'local') {
|
||||
const docsBuilder = new DocsBuilder({
|
||||
preparers,
|
||||
generators,
|
||||
publisher,
|
||||
dockerClient,
|
||||
logger,
|
||||
entity,
|
||||
});
|
||||
if (publisherType === 'local') {
|
||||
if (!(await docsBuilder.docsUpToDate())) {
|
||||
await docsBuilder.build();
|
||||
}
|
||||
} else if (publisherType === 'googleGcs') {
|
||||
// This block should be valid for all external storage implementations. So no need to duplicate in future,
|
||||
// add the publisher type in the list here.
|
||||
if (!(await publisher.hasDocsBeenGenerated(entity))) {
|
||||
logger.info(
|
||||
'No pre-generated documentation files found for the entity in the storage. Building docs...',
|
||||
);
|
||||
await docsBuilder.build();
|
||||
// With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched
|
||||
// on the user's page. If not, respond with a message asking them to check back later.
|
||||
// The delay here is to make sure GCS registers newly uploaded files which is usually <1 second
|
||||
let foundDocs = false;
|
||||
for (let attempt = 0; attempt < 5; attempt++) {
|
||||
if (await publisher.hasDocsBeenGenerated(entity)) {
|
||||
foundDocs = true;
|
||||
break;
|
||||
}
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
if (!foundDocs) {
|
||||
logger.error(
|
||||
'Published files are taking longer to show up in storage. Something went wrong.',
|
||||
);
|
||||
res
|
||||
.status(408)
|
||||
.send(
|
||||
'Sorry! It is taking longer for the generated docs to show up in storage. Check back later.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
logger.info(
|
||||
'Found pre-generated docs for this entity. Serving them.',
|
||||
);
|
||||
// TODO: re-trigger build for cache invalidation.
|
||||
// Compare the date modified of the requested file on storage and compare it against
|
||||
// the last modified or last commit timestamp in the repository.
|
||||
// Without this, docs will not be re-built once they have been generated.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`);
|
||||
});
|
||||
|
||||
if (publisher instanceof LocalPublish) {
|
||||
router.use('/static/docs', express.static(staticDocsDir));
|
||||
}
|
||||
// Route middleware which serves files from the storage set in the publisher.
|
||||
router.use('/static/docs', publisher.docsRouter());
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
@@ -27,8 +27,8 @@ import {
|
||||
DirectoryPreparer,
|
||||
Generators,
|
||||
TechdocsGenerator,
|
||||
LocalPublish,
|
||||
} from '../techdocs';
|
||||
Publisher,
|
||||
} from '@backstage/techdocs-common';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
|
||||
export interface ServerOptions {
|
||||
@@ -41,7 +41,18 @@ export async function startStandaloneServer(
|
||||
options: ServerOptions,
|
||||
): Promise<Server> {
|
||||
const logger = options.logger.child({ service: 'techdocs-backend' });
|
||||
const config = ConfigReader.fromConfigs([]);
|
||||
const config = ConfigReader.fromConfigs([
|
||||
{
|
||||
context: '',
|
||||
data: {
|
||||
techdocs: {
|
||||
publisher: {
|
||||
type: 'local',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
const discovery = SingleHostDiscovery.fromConfig(config);
|
||||
|
||||
logger.debug('Creating application...');
|
||||
@@ -53,7 +64,7 @@ export async function startStandaloneServer(
|
||||
const techdocsGenerator = new TechdocsGenerator(logger, config);
|
||||
generators.register('techdocs', techdocsGenerator);
|
||||
|
||||
const publisher = new LocalPublish(logger, discovery);
|
||||
const publisher = Publisher.fromConfig(config, logger, discovery);
|
||||
|
||||
const dockerClient = new Docker();
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export * from './stages';
|
||||
@@ -1,2 +0,0 @@
|
||||
site_name: Test site name
|
||||
site_description: Test site description
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
site_name: Test site name
|
||||
site_description: Test site description
|
||||
|
||||
repo_url: https://github.com/backstage/backstage
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Generators, TechdocsGenerator } from './';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const mockEntity = {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
},
|
||||
};
|
||||
|
||||
describe('generators', () => {
|
||||
it('should return error if no generator is registered', async () => {
|
||||
const generators = new Generators();
|
||||
|
||||
expect(() => generators.get(mockEntity)).toThrowError(
|
||||
'No generator registered for entity: "techdocs"',
|
||||
);
|
||||
});
|
||||
|
||||
it('should return correct registered generator', async () => {
|
||||
const generators = new Generators();
|
||||
const techdocs = new TechdocsGenerator(
|
||||
logger,
|
||||
ConfigReader.fromConfigs([]),
|
||||
);
|
||||
|
||||
generators.register('techdocs', techdocs);
|
||||
|
||||
expect(generators.get(mockEntity)).toBe(techdocs);
|
||||
});
|
||||
});
|
||||
@@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
GeneratorBase,
|
||||
SupportedGeneratorKey,
|
||||
GeneratorBuilder,
|
||||
} from './types';
|
||||
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { getGeneratorKey } from './helpers';
|
||||
|
||||
export class Generators implements GeneratorBuilder {
|
||||
private generatorMap = new Map<SupportedGeneratorKey, GeneratorBase>();
|
||||
|
||||
register(generatorKey: SupportedGeneratorKey, generator: GeneratorBase) {
|
||||
this.generatorMap.set(generatorKey, generator);
|
||||
}
|
||||
|
||||
get(entity: Entity): GeneratorBase {
|
||||
const generatorKey = getGeneratorKey(entity);
|
||||
const generator = this.generatorMap.get(generatorKey);
|
||||
|
||||
if (!generator) {
|
||||
throw new Error(`No generator registered for entity: "${generatorKey}"`);
|
||||
}
|
||||
|
||||
return generator;
|
||||
}
|
||||
}
|
||||
@@ -1,332 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import Stream, { PassThrough } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import mockFs from 'mock-fs';
|
||||
import * as winston from 'winston';
|
||||
import {
|
||||
runDockerContainer,
|
||||
getGeneratorKey,
|
||||
isValidRepoUrlForMkdocs,
|
||||
getRepoUrlFromLocationAnnotation,
|
||||
patchMkdocsYmlPreBuild,
|
||||
} from './helpers';
|
||||
import { RemoteProtocol } from '../prepare/types';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
|
||||
const mockEntity = {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
},
|
||||
};
|
||||
|
||||
const mockDocker = new Docker() as jest.Mocked<Docker>;
|
||||
|
||||
const mkdocsYml = fs.readFileSync(
|
||||
resolvePath(__filename, '../__fixtures__/mkdocs.yml'),
|
||||
);
|
||||
const mkdocsYmlWithRepoUrl = fs.readFileSync(
|
||||
resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'),
|
||||
);
|
||||
const mockLogger = winston.createLogger();
|
||||
|
||||
describe('helpers', () => {
|
||||
describe('getGeneratorKey', () => {
|
||||
it('should return techdocs as the only generator key', () => {
|
||||
const key = getGeneratorKey(mockEntity);
|
||||
expect(key).toBe('techdocs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runDockerContainer', () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(mockDocker, 'pull').mockImplementation((async (
|
||||
_image: string,
|
||||
_something: any,
|
||||
handler: (err: Error | undefined, stream: PassThrough) => void,
|
||||
) => {
|
||||
const mockStream = new PassThrough();
|
||||
handler(undefined, mockStream);
|
||||
mockStream.end();
|
||||
}) as any);
|
||||
|
||||
jest
|
||||
.spyOn(mockDocker, 'run')
|
||||
.mockResolvedValue([{ Error: null, StatusCode: 0 }]);
|
||||
|
||||
jest
|
||||
.spyOn(mockDocker, 'ping')
|
||||
.mockResolvedValue(Buffer.from('OK', 'utf-8'));
|
||||
});
|
||||
|
||||
const imageName = 'spotify/techdocs';
|
||||
const args = ['build', '-d', '/result'];
|
||||
const docsDir = os.tmpdir();
|
||||
const resultDir = os.tmpdir();
|
||||
|
||||
it('should pull the techdocs docker container', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.pull).toHaveBeenCalledWith(
|
||||
imageName,
|
||||
{},
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it('should run the techdocs docker container', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.run).toHaveBeenCalledWith(
|
||||
imageName,
|
||||
args,
|
||||
expect.any(Stream),
|
||||
{
|
||||
Volumes: {
|
||||
'/content': {},
|
||||
'/result': {},
|
||||
},
|
||||
WorkingDir: '/content',
|
||||
HostConfig: {
|
||||
Binds: [`${docsDir}:/content`, `${resultDir}:/result`],
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('should ping docker to test availability', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.ping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('where docker is unavailable', () => {
|
||||
const dockerError = 'a docker error';
|
||||
|
||||
beforeEach(() => {
|
||||
jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => {
|
||||
throw new Error(dockerError);
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw with a descriptive error message including the docker error message', async () => {
|
||||
await expect(
|
||||
runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
}),
|
||||
).rejects.toThrow(new RegExp(`.+: ${dockerError}`));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidRepoUrlForMkdocs', () => {
|
||||
it('should return true for valid repo_url values for mkdocs', () => {
|
||||
const validRepoUrls = [
|
||||
'https://github.com/org/repo',
|
||||
'https://github.com/backstage/backstage/',
|
||||
'https://github.com/org123/repo1-2-3/',
|
||||
'http://github.com/insecureOrg/insecureRepo',
|
||||
'https://gitlab.com/org/repo',
|
||||
'https://gitlab.com/backstage/backstage/',
|
||||
'https://gitlab.com/org123/repo1-2-3/',
|
||||
'http://gitlab.com/insecureOrg/insecureRepo',
|
||||
];
|
||||
|
||||
const validRemoteProtocols = ['github', 'gitlab'];
|
||||
|
||||
validRepoUrls.forEach(url => {
|
||||
validRemoteProtocols.forEach(targetType => {
|
||||
expect(
|
||||
isValidRepoUrlForMkdocs(url, targetType as RemoteProtocol),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should return false for invalid repo_urls values for mkdocs', () => {
|
||||
const invalidRepoUrls = [
|
||||
'git@github.com:org/repo',
|
||||
'https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend',
|
||||
];
|
||||
|
||||
invalidRepoUrls.forEach(url => {
|
||||
expect(isValidRepoUrlForMkdocs(url, 'github')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it('should return false for unsupported remote protocols', () => {
|
||||
const validRepoUrl = 'https://github.com/backstage/backstage';
|
||||
|
||||
const unsupportedRemoteProtocols = ['dir', 'file', 'url'];
|
||||
|
||||
unsupportedRemoteProtocols.forEach(targetType => {
|
||||
expect(
|
||||
isValidRepoUrlForMkdocs(validRepoUrl, targetType as RemoteProtocol),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRepoUrlFromLocationAnnotation', () => {
|
||||
it('should return undefined for unsupported location type', () => {
|
||||
const parsedLocationAnnotation1: ParsedLocationAnnotation = {
|
||||
type: 'dir',
|
||||
target: '/home/user/workspace/docs-repository',
|
||||
};
|
||||
|
||||
const parsedLocationAnnotation2: ParsedLocationAnnotation = {
|
||||
type: 'file',
|
||||
target: '/home/user/workspace/docs-repository/catalog-info.yaml',
|
||||
};
|
||||
|
||||
const parsedLocationAnnotation3: ParsedLocationAnnotation = {
|
||||
type: 'url',
|
||||
target: 'https://my-website.com/storage/this/docs/repository',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
|
||||
undefined,
|
||||
);
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return correct target url for supported hosts', () => {
|
||||
const parsedLocationAnnotation1: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/backstage/backstage.git',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
|
||||
'https://github.com/backstage/backstage',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation2: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/org/repo',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
|
||||
'https://github.com/org/repo',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation3: ParsedLocationAnnotation = {
|
||||
type: 'gitlab',
|
||||
target: 'https://gitlab.com/org/repo',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
|
||||
'https://gitlab.com/org/repo',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation4: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target:
|
||||
'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation4)).toBe(
|
||||
'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pathMkdocsPreBuild', () => {
|
||||
beforeEach(() => {
|
||||
mockFs({
|
||||
'/mkdocs.yml': mkdocsYml,
|
||||
'/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should add repo_url to mkdocs.yml', async () => {
|
||||
const parsedLocationAnnotation: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/backstage/backstage',
|
||||
};
|
||||
|
||||
await patchMkdocsYmlPreBuild(
|
||||
'/mkdocs.yml',
|
||||
mockLogger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
const updatedMkdocsYml = await fs.readFile('/mkdocs.yml');
|
||||
|
||||
expect(updatedMkdocsYml.toString()).toContain(
|
||||
"repo_url: 'https://github.com/backstage/backstage'",
|
||||
);
|
||||
});
|
||||
|
||||
it('should not override existing repo_url in mkdocs.yml', async () => {
|
||||
const parsedLocationAnnotation: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/neworg/newrepo',
|
||||
};
|
||||
|
||||
await patchMkdocsYmlPreBuild(
|
||||
'/mkdocs_with_repo_url.yml',
|
||||
mockLogger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
const updatedMkdocsYml = await fs.readFile('/mkdocs_with_repo_url.yml');
|
||||
|
||||
expect(updatedMkdocsYml.toString()).toContain(
|
||||
"repo_url: 'https://github.com/backstage/backstage'",
|
||||
);
|
||||
expect(updatedMkdocsYml.toString()).not.toContain(
|
||||
"repo_url: 'https://github.com/neworg/newrepo'",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,277 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import { spawn } from 'child_process';
|
||||
import { Writable, PassThrough } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import yaml from 'js-yaml';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { SupportedGeneratorKey } from './types';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
import { RemoteProtocol } from '../prepare/types';
|
||||
|
||||
// TODO: Implement proper support for more generators.
|
||||
export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
|
||||
if (!entity) {
|
||||
throw new Error('No entity provided');
|
||||
}
|
||||
|
||||
return 'techdocs';
|
||||
}
|
||||
|
||||
type RunDockerContainerOptions = {
|
||||
imageName: string;
|
||||
args: string[];
|
||||
logStream?: Writable;
|
||||
docsDir: string;
|
||||
resultDir: string;
|
||||
dockerClient: Docker;
|
||||
createOptions?: Docker.ContainerCreateOptions;
|
||||
};
|
||||
|
||||
export type RunCommandOptions = {
|
||||
command: string;
|
||||
args: string[];
|
||||
options: object;
|
||||
logStream?: Writable;
|
||||
};
|
||||
|
||||
export async function runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
logStream = new PassThrough(),
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient,
|
||||
createOptions,
|
||||
}: RunDockerContainerOptions) {
|
||||
try {
|
||||
await dockerClient.ping();
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
dockerClient.pull(imageName, {}, (err, stream) => {
|
||||
if (err) return reject(err);
|
||||
stream.pipe(logStream, { end: false });
|
||||
stream.on('end', () => resolve());
|
||||
stream.on('error', (error: Error) => reject(error));
|
||||
return undefined;
|
||||
});
|
||||
});
|
||||
|
||||
const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
|
||||
imageName,
|
||||
args,
|
||||
logStream,
|
||||
{
|
||||
Volumes: {
|
||||
'/content': {},
|
||||
'/result': {},
|
||||
},
|
||||
WorkingDir: '/content',
|
||||
HostConfig: {
|
||||
Binds: [`${docsDir}:/content`, `${resultDir}:/result`],
|
||||
},
|
||||
...createOptions,
|
||||
},
|
||||
);
|
||||
|
||||
if (error) {
|
||||
throw new Error(
|
||||
`Docker failed to run with the following error message: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (statusCode !== 0) {
|
||||
throw new Error(
|
||||
`Docker container returned a non-zero exit code (${statusCode})`,
|
||||
);
|
||||
}
|
||||
|
||||
return { error, statusCode };
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param options the options object
|
||||
* @param options.command the command to run
|
||||
* @param options.args the arguments to pass the command
|
||||
* @param options.options options used in spawn
|
||||
* @param options.logStream the log streamer to capture log messages
|
||||
*/
|
||||
export const runCommand = async ({
|
||||
command,
|
||||
args,
|
||||
options,
|
||||
logStream = new PassThrough(),
|
||||
}: RunCommandOptions) => {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const process = spawn(command, args, options);
|
||||
|
||||
process.stdout.on('data', stream => {
|
||||
logStream.write(stream);
|
||||
});
|
||||
|
||||
process.stderr.on('data', stream => {
|
||||
logStream.write(stream);
|
||||
});
|
||||
|
||||
process.on('error', error => {
|
||||
return reject(error);
|
||||
});
|
||||
|
||||
process.on('close', code => {
|
||||
if (code !== 0) {
|
||||
return reject(`Command ${command} failed, exit code: ${code}`);
|
||||
}
|
||||
return resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Return true if mkdocs can compile docs with provided repo_url
|
||||
*
|
||||
* Valid repo_url examples in mkdocs.yml
|
||||
* - https://github.com/backstage/backstage
|
||||
* - https://gitlab.com/org/repo/
|
||||
* - http://github.com/backstage/backstage
|
||||
* - A http(s) protocol URL to the root of the repository
|
||||
*
|
||||
* Invalid repo_url examples in mkdocs.yml
|
||||
* - https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component
|
||||
* - (anything that is not valid as described above)
|
||||
*
|
||||
* @param {string} repoUrl URL supposed to be used as repo_url in mkdocs.yml
|
||||
* @param {RemoteProtocol} locationType Type of source code host - github, gitlab, dir, url, etc.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export const isValidRepoUrlForMkdocs = (
|
||||
repoUrl: string,
|
||||
locationType: RemoteProtocol,
|
||||
): boolean => {
|
||||
// Trim trailing slash
|
||||
const cleanRepoUrl = repoUrl.replace(/\/$/, '');
|
||||
|
||||
if (locationType === 'github' || locationType === 'gitlab') {
|
||||
// A valid repoUrl to the root of the repository will be split into 5 strings if split using the / delimiter.
|
||||
// We do not want URLs which have more than that number of forward slashes since they will signify a non-root location
|
||||
// Note: This is not the best possible implementation but will work most of the times.. Feel free to improve or
|
||||
// highlight edge cases.
|
||||
return cleanRepoUrl.split('/').length === 5;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a valid URL of the repository used in backstage.io/techdocs-ref annotation.
|
||||
* Return undefined if the `target` is not valid in context of repo_url in mkdocs.yml
|
||||
* Alter URL so that it is a valid repo_url config in mkdocs.yml
|
||||
*
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
|
||||
* @returns {string | undefined}
|
||||
*/
|
||||
export const getRepoUrlFromLocationAnnotation = (
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation,
|
||||
): string | undefined => {
|
||||
const { type: locationType, target } = parsedLocationAnnotation;
|
||||
|
||||
// Add more options from the RemoteProtocol type of parsedLocationAnnotation.type here
|
||||
// when TechDocs supports more hosts and if mkdocs can generated an Edit URL for them.
|
||||
const supportedHosts = ['github', 'gitlab'];
|
||||
|
||||
if (supportedHosts.includes(locationType)) {
|
||||
// Trim .git or .git/ from the end of repository url
|
||||
return target.replace(/.git\/*$/, '');
|
||||
}
|
||||
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the mkdocs.yml file before TechDocs generator uses it to build docs site.
|
||||
*
|
||||
* List of tasks:
|
||||
* - Add repo_url if it does not exists
|
||||
* If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default.
|
||||
* If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get
|
||||
* the repository URL.
|
||||
*
|
||||
* This function will not throw an error since this is not critical to the whole TechDocs pipeline.
|
||||
* Instead it will log warnings if there are any errors in reading, parsing or writing YAML.
|
||||
*
|
||||
* @param {string} mkdocsYmlPath Absolute path to mkdocs.yml or equivalent of a docs site
|
||||
* @param {Logger} logger
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
|
||||
*/
|
||||
export const patchMkdocsYmlPreBuild = async (
|
||||
mkdocsYmlPath: string,
|
||||
logger: Logger,
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation,
|
||||
) => {
|
||||
let mkdocsYmlFileString;
|
||||
try {
|
||||
mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8');
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Could not read file ${mkdocsYmlPath} before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mkdocsYml: any;
|
||||
try {
|
||||
mkdocsYml = yaml.safeLoad(mkdocsYmlFileString);
|
||||
|
||||
// mkdocsYml should be an object type after successful parsing.
|
||||
// But based on its type definition, it can also be a string or undefined, which we don't want.
|
||||
if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') {
|
||||
throw new Error('Bad YAML format.');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add repo_url to mkdocs.yml if it is missing. This will enable the Page edit button generated by MkDocs.
|
||||
if (!('repo_url' in mkdocsYml)) {
|
||||
const repoUrl = getRepoUrlFromLocationAnnotation(parsedLocationAnnotation);
|
||||
if (repoUrl !== undefined) {
|
||||
// mkdocs.yml will not build with invalid repo_url. So, make sure it is valid.
|
||||
if (isValidRepoUrlForMkdocs(repoUrl, parsedLocationAnnotation.type)) {
|
||||
mkdocsYml.repo_url = repoUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.writeFile(mkdocsYmlPath, yaml.safeDump(mkdocsYml), 'utf8');
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -1,18 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { TechdocsGenerator } from './techdocs';
|
||||
export { Generators } from './generators';
|
||||
export type { GeneratorBuilder, GeneratorBase } from './types';
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { Logger } from 'winston';
|
||||
import { PassThrough } from 'stream';
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
import {
|
||||
GeneratorBase,
|
||||
GeneratorRunOptions,
|
||||
GeneratorRunResult,
|
||||
} from './types';
|
||||
import {
|
||||
runDockerContainer,
|
||||
runCommand,
|
||||
patchMkdocsYmlPreBuild,
|
||||
} from './helpers';
|
||||
|
||||
type TechdocsGeneratorOptions = {
|
||||
// This option enables users to configure if they want to use TechDocs container
|
||||
// or generate without the container.
|
||||
// This is used to avoid running into Docker in Docker environment.
|
||||
runGeneratorIn: string;
|
||||
};
|
||||
|
||||
const createStream = (): [string[], PassThrough] => {
|
||||
const log = [] as Array<string>;
|
||||
|
||||
const stream = new PassThrough();
|
||||
stream.on('data', chunk => {
|
||||
const textValue = chunk.toString().trim();
|
||||
if (textValue?.length > 1) log.push(textValue);
|
||||
});
|
||||
|
||||
return [log, stream];
|
||||
};
|
||||
|
||||
export class TechdocsGenerator implements GeneratorBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly options: TechdocsGeneratorOptions;
|
||||
|
||||
constructor(logger: Logger, config: Config) {
|
||||
this.logger = logger;
|
||||
this.options = {
|
||||
runGeneratorIn:
|
||||
config.getOptionalString('techdocs.generators.techdocs') ?? 'docker',
|
||||
};
|
||||
}
|
||||
|
||||
public async run({
|
||||
directory,
|
||||
dockerClient,
|
||||
parsedLocationAnnotation,
|
||||
}: GeneratorRunOptions): Promise<GeneratorRunResult> {
|
||||
const tmpdirPath = os.tmpdir();
|
||||
// Fixes a problem with macOS returning a path that is a symlink
|
||||
const tmpdirResolvedPath = fs.realpathSync(tmpdirPath);
|
||||
const resultDir = fs.mkdtempSync(
|
||||
path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
|
||||
);
|
||||
const [log, logStream] = createStream();
|
||||
|
||||
// TODO: In future mkdocs.yml can be mkdocs.yaml. So, use a config variable here to find out
|
||||
// the correct file name.
|
||||
// Do some updates to mkdocs.yml before generating docs e.g. adding repo_url
|
||||
await patchMkdocsYmlPreBuild(
|
||||
path.join(directory, 'mkdocs.yml'),
|
||||
this.logger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
try {
|
||||
switch (this.options.runGeneratorIn) {
|
||||
case 'local':
|
||||
await runCommand({
|
||||
command: 'mkdocs',
|
||||
args: ['build', '-d', resultDir, '-v'],
|
||||
options: {
|
||||
cwd: directory,
|
||||
},
|
||||
logStream,
|
||||
});
|
||||
this.logger.info(
|
||||
`Successfully generated docs from ${directory} into ${resultDir} using local mkdocs`,
|
||||
);
|
||||
break;
|
||||
case 'docker':
|
||||
await runDockerContainer({
|
||||
imageName: 'spotify/techdocs',
|
||||
args: ['build', '-d', '/result'],
|
||||
logStream,
|
||||
docsDir: directory,
|
||||
resultDir,
|
||||
dockerClient,
|
||||
});
|
||||
this.logger.info(
|
||||
`Successfully generated docs from ${directory} into ${resultDir} using techdocs-container`,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.debug(
|
||||
`Failed to generate docs from ${directory} into ${resultDir}`,
|
||||
);
|
||||
this.logger.debug(`Build failed with error: ${log}`);
|
||||
throw new Error(
|
||||
`Failed to generate docs from ${directory} into ${resultDir} with error ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
return { resultDir };
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Writable } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
|
||||
/**
|
||||
* The returned directory from the generator which is ready
|
||||
* to pass to the next stage of the TechDocs which is publishing
|
||||
*/
|
||||
export type GeneratorRunResult = {
|
||||
resultDir: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The values that the generator will receive.
|
||||
*
|
||||
* @param {string} directory The directory of the uncompiled documentation, with the values from the frontend
|
||||
* @param {Docker} dockerClient A docker client to run any generator on top of your directory
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity
|
||||
* @param {Writable} [logStream] A dedicated log stream
|
||||
*/
|
||||
export type GeneratorRunOptions = {
|
||||
directory: string;
|
||||
dockerClient: Docker;
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation;
|
||||
logStream?: Writable;
|
||||
};
|
||||
|
||||
export type GeneratorBase = {
|
||||
// runs the generator with the values and returns the directory to be published
|
||||
run(opts: GeneratorRunOptions): Promise<GeneratorRunResult>;
|
||||
};
|
||||
|
||||
/**
|
||||
* List of supported generator options
|
||||
*/
|
||||
export type SupportedGeneratorKey = 'techdocs' | string;
|
||||
|
||||
/**
|
||||
* The generator builder holds the generator ready for run time
|
||||
*/
|
||||
export type GeneratorBuilder = {
|
||||
register(protocol: SupportedGeneratorKey, generator: GeneratorBase): void;
|
||||
get(entity: Entity): GeneratorBase;
|
||||
};
|
||||
@@ -1,93 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { CommonGitPreparer } from './commonGit';
|
||||
import { checkoutGitRepository } from '../../../helpers';
|
||||
|
||||
function normalizePath(path: string) {
|
||||
return path
|
||||
.replace(/^[a-z]:/i, '')
|
||||
.split('\\')
|
||||
.join('/');
|
||||
}
|
||||
|
||||
jest.mock('../../../helpers', () => ({
|
||||
...jest.requireActual<{}>('../../../helpers'),
|
||||
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
|
||||
}));
|
||||
|
||||
const createMockEntity = (annotations = {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'test-component-name',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('commonGit preparer', () => {
|
||||
it('should prepare temp docs path from github repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'github:https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component',
|
||||
);
|
||||
});
|
||||
|
||||
it('should prepare temp docs path from gitlab repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(2);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/catalog-info.yaml',
|
||||
);
|
||||
});
|
||||
|
||||
it('should prepare temp docs path from azure repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(3);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/template.yaml',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,50 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PreparerBase } from './types';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import {
|
||||
parseReferenceAnnotation,
|
||||
checkoutGitRepository,
|
||||
} from '../../../helpers';
|
||||
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export class CommonGitPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
try {
|
||||
const repoPath = await checkoutGitRepository(target, this.logger);
|
||||
const parsedGitLocation = parseGitUrl(target);
|
||||
|
||||
return path.join(repoPath, parsedGitLocation.filepath);
|
||||
} catch (error) {
|
||||
this.logger.debug(`Repo checkout failed with error ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { DirectoryPreparer } from './dir';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { checkoutGitRepository } from '../../../helpers';
|
||||
|
||||
function normalizePath(path: string) {
|
||||
return path
|
||||
.replace(/^[a-z]:/i, '')
|
||||
.split('\\')
|
||||
.join('/');
|
||||
}
|
||||
|
||||
jest.mock('../../../helpers', () => ({
|
||||
...jest.requireActual<{}>('../../../helpers'),
|
||||
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
|
||||
}));
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const createMockEntity = (annotations: {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
describe('directory preparer', () => {
|
||||
it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'file:/directory/documented-component.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:./our-documentation',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/directory/our-documentation',
|
||||
);
|
||||
});
|
||||
|
||||
it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'file:/directory/documented-component.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/our-documentation/techdocs',
|
||||
);
|
||||
});
|
||||
|
||||
it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'github:https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:./docs',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/docs',
|
||||
);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,77 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { PreparerBase } from './types';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import path from 'path';
|
||||
import {
|
||||
parseReferenceAnnotation,
|
||||
checkoutGitRepository,
|
||||
} from '../../../helpers';
|
||||
import { InputError } from '@backstage/backend-common';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export class DirectoryPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
private async resolveManagedByLocationToDir(entity: Entity) {
|
||||
const { type, target } = parseReferenceAnnotation(
|
||||
'backstage.io/managed-by-location',
|
||||
entity,
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Building docs for entity with type 'dir' and managed-by-location '${type}'`,
|
||||
);
|
||||
switch (type) {
|
||||
case 'github':
|
||||
case 'gitlab':
|
||||
case 'url':
|
||||
case 'azure/api': {
|
||||
const parsedGitLocation = parseGitUrl(target);
|
||||
const repoLocation = await checkoutGitRepository(target, this.logger);
|
||||
|
||||
return path.dirname(
|
||||
path.join(repoLocation, parsedGitLocation.filepath),
|
||||
);
|
||||
}
|
||||
|
||||
case 'file':
|
||||
return path.dirname(target);
|
||||
default:
|
||||
throw new InputError(`Unable to resolve location type ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
const managedByLocationDirectory = await this.resolveManagedByLocationToDir(
|
||||
entity,
|
||||
);
|
||||
|
||||
return new Promise(resolve => {
|
||||
resolve(path.resolve(managedByLocationDirectory, target));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { DirectoryPreparer } from './dir';
|
||||
export { CommonGitPreparer } from './commonGit';
|
||||
export { UrlPreparer } from './url';
|
||||
export { Preparers } from './preparers';
|
||||
export type { PreparerBuilder, PreparerBase } from './types';
|
||||
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { parseReferenceAnnotation } from '../../../helpers';
|
||||
|
||||
export class Preparers implements PreparerBuilder {
|
||||
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
|
||||
|
||||
register(protocol: RemoteProtocol, preparer: PreparerBase) {
|
||||
this.preparerMap.set(protocol, preparer);
|
||||
}
|
||||
|
||||
get(entity: Entity): PreparerBase {
|
||||
const { type } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
const preparer = this.preparerMap.get(type);
|
||||
|
||||
if (!preparer) {
|
||||
throw new Error(`No preparer registered for type: "${type}"`);
|
||||
}
|
||||
|
||||
return preparer;
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import type { Entity } from '@backstage/catalog-model';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export type PreparerBase = {
|
||||
/**
|
||||
* Given an Entity definition from the Service Catalog, go and prepare a directory
|
||||
* with contents from the location in temporary storage and return the path
|
||||
* @param entity The entity from the Service Catalog
|
||||
*/
|
||||
prepare(entity: Entity, opts?: { logger: Logger }): Promise<string>;
|
||||
};
|
||||
|
||||
export type PreparerBuilder = {
|
||||
register(protocol: RemoteProtocol, preparer: PreparerBase): void;
|
||||
get(entity: Entity): PreparerBase;
|
||||
};
|
||||
|
||||
export type RemoteProtocol =
|
||||
| 'dir'
|
||||
| 'github'
|
||||
| 'gitlab'
|
||||
| 'file'
|
||||
| 'azure/api'
|
||||
| 'url';
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PreparerBase } from './types';
|
||||
import { getDocFilesFromRepository } from '../../../helpers';
|
||||
|
||||
import { Logger } from 'winston';
|
||||
import { UrlReader } from '@backstage/backend-common';
|
||||
|
||||
export class UrlPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly reader: UrlReader;
|
||||
|
||||
constructor(reader: UrlReader, logger: Logger) {
|
||||
this.logger = logger;
|
||||
this.reader = reader;
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
try {
|
||||
return getDocFilesFromRepository(this.reader, entity);
|
||||
} catch (error) {
|
||||
this.logger.debug(
|
||||
`Unable to fetch files for building docs ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { LocalPublish } from './local';
|
||||
export type { PublisherBase } from './types';
|
||||
@@ -1,75 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-restricted-syntax */
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import {
|
||||
getVoidLogger,
|
||||
PluginEndpointDiscovery,
|
||||
} from '@backstage/backend-common';
|
||||
import { LocalPublish } from './local';
|
||||
|
||||
const createMockEntity = (annotations = {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'test-component-name',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('local publisher', () => {
|
||||
it('should publish generated documentation dir', async () => {
|
||||
const testDiscovery: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7000'),
|
||||
getExternalBaseUrl: jest.fn(),
|
||||
};
|
||||
|
||||
const publisher = new LocalPublish(logger, testDiscovery);
|
||||
|
||||
const mockEntity = createMockEntity();
|
||||
|
||||
const tempDir = fs.mkdtempSync(`${__dirname}/test-component-folder-`);
|
||||
|
||||
expect(tempDir).toBeTruthy();
|
||||
|
||||
fs.closeSync(fs.openSync(path.join(tempDir, '/mock-file'), 'w'));
|
||||
|
||||
await publisher.publish({ entity: mockEntity, directory: tempDir });
|
||||
const publishDir = path.resolve(
|
||||
__dirname,
|
||||
`../../../../static/docs/${mockEntity.metadata.name}`,
|
||||
);
|
||||
|
||||
const resultDir = path.resolve(
|
||||
__dirname,
|
||||
`../../../../static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`,
|
||||
);
|
||||
|
||||
expect(fs.existsSync(resultDir)).toBeTruthy();
|
||||
expect(fs.existsSync(path.join(resultDir, '/mock-file'))).toBeTruthy();
|
||||
|
||||
fs.removeSync(publishDir);
|
||||
fs.removeSync(tempDir);
|
||||
});
|
||||
});
|
||||
@@ -1,82 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs-extra';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PublisherBase } from './types';
|
||||
import {
|
||||
resolvePackagePath,
|
||||
PluginEndpointDiscovery,
|
||||
} from '@backstage/backend-common';
|
||||
|
||||
export class LocalPublish implements PublisherBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly discovery: PluginEndpointDiscovery;
|
||||
|
||||
constructor(logger: Logger, discovery: PluginEndpointDiscovery) {
|
||||
this.logger = logger;
|
||||
this.discovery = discovery;
|
||||
}
|
||||
|
||||
publish({
|
||||
entity,
|
||||
directory,
|
||||
}: {
|
||||
entity: Entity;
|
||||
directory: string;
|
||||
}):
|
||||
| Promise<{
|
||||
remoteUrl: string;
|
||||
}>
|
||||
| { remoteUrl: string } {
|
||||
const entityNamespace = entity.metadata.namespace ?? 'default';
|
||||
|
||||
const publishDir = resolvePackagePath(
|
||||
'@backstage/plugin-techdocs-backend',
|
||||
'static/docs',
|
||||
entityNamespace,
|
||||
entity.kind,
|
||||
entity.metadata.name,
|
||||
);
|
||||
|
||||
if (!fs.existsSync(publishDir)) {
|
||||
this.logger.info(`Could not find ${publishDir}, creating the directory.`);
|
||||
fs.mkdirSync(publishDir, { recursive: true });
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.copy(directory, publishDir, err => {
|
||||
if (err) {
|
||||
this.logger.debug(
|
||||
`Failed to copy docs from ${directory} to ${publishDir}`,
|
||||
);
|
||||
reject(err);
|
||||
}
|
||||
|
||||
this.discovery
|
||||
.getBaseUrl('techdocs')
|
||||
.then(techdocsApiUrl => {
|
||||
resolve({
|
||||
remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
|
||||
});
|
||||
})
|
||||
.catch(reason => {
|
||||
reject(reason);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,7 @@ Set up Backstage and TechDocs by follow our guide on [Getting Started](../../doc
|
||||
|
||||
## Configuration
|
||||
|
||||
### Custom Storage URL
|
||||
|
||||
TechDocs will try to read your documentation from the URL you have specified in the `techdocs storageUrl` in `app-config.yml`.
|
||||
http://backstage.io/docs/features/techdocs/configuration
|
||||
|
||||
### TechDocs Storage Api
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
"@backstage/plugin-catalog": "^0.2.6",
|
||||
"@backstage/test-utils": "^0.1.5",
|
||||
"@backstage/theme": "^0.2.2",
|
||||
"@backstage/techdocs-common": "^0.1.1",
|
||||
"@material-ui/core": "^4.11.0",
|
||||
"@material-ui/icons": "^4.9.1",
|
||||
"@material-ui/lab": "4.0.0-alpha.45",
|
||||
@@ -77,11 +78,68 @@
|
||||
"visibility": "frontend"
|
||||
},
|
||||
"storageUrl": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"visibility": "backend"
|
||||
},
|
||||
"generators": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"techdocs": {
|
||||
"type": "string",
|
||||
"visibility": "backend"
|
||||
}
|
||||
}
|
||||
},
|
||||
"builder": {
|
||||
"type": "string",
|
||||
"visibility": "frontend"
|
||||
},
|
||||
"publisher": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "local",
|
||||
"visibility": "backend"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "googleGcs",
|
||||
"visibility": "backend"
|
||||
},
|
||||
"googleGcs": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"credentials": {
|
||||
"type": "string",
|
||||
"visibility": "secret"
|
||||
},
|
||||
"projectId": {
|
||||
"type": "string",
|
||||
"visibility": "secret"
|
||||
},
|
||||
"bucketName": {
|
||||
"type": "string",
|
||||
"visibility": "secret"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"requestUrl"
|
||||
"requestUrl",
|
||||
"storageUrl",
|
||||
"builder"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,18 +15,31 @@
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { ErrorPage } from '@backstage/core';
|
||||
import { ErrorPage, useApi, configApiRef } from '@backstage/core';
|
||||
|
||||
type Props = {
|
||||
errorMessage?: string;
|
||||
};
|
||||
|
||||
export const TechDocsNotFound = ({ errorMessage }: Props) => {
|
||||
const techdocsBuilder = useApi(configApiRef).getOptionalString(
|
||||
'techdocs.builder',
|
||||
);
|
||||
|
||||
let additionalInfo = '';
|
||||
if (techdocsBuilder !== 'local') {
|
||||
additionalInfo =
|
||||
"Note that techdocs.builder is not set to 'local' in your config, which means this Backstage app will not " +
|
||||
"generate docs if they are not found. Make sure the project's docs are generated and published by some external " +
|
||||
"process (e.g. CI/CD pipeline). Or change techdocs.builder to 'local' to generate docs from this Backstage " +
|
||||
'instance.';
|
||||
}
|
||||
|
||||
return (
|
||||
<ErrorPage
|
||||
status="404"
|
||||
statusMessage={errorMessage || 'Documentation not found'}
|
||||
additionalInfo={window.location.pathname}
|
||||
additionalInfo={additionalInfo}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
import React from 'react';
|
||||
import { AsyncState } from 'react-use/lib/useAsync';
|
||||
import { CircularProgress } from '@material-ui/core';
|
||||
import CodeIcon from '@material-ui/icons/Code';
|
||||
import { EntityName } from '@backstage/catalog-model';
|
||||
import { Header, HeaderLabel, Link } from '@backstage/core';
|
||||
@@ -86,7 +85,7 @@ export const TechDocsPageHeader = ({
|
||||
|
||||
return (
|
||||
<Header
|
||||
title={siteName ? siteName : <CircularProgress />}
|
||||
title={siteName ? siteName : '.'}
|
||||
pageTitleOverride={siteName || name}
|
||||
subtitle={
|
||||
siteDescription && siteDescription !== 'None' ? siteDescription : ''
|
||||
|
||||
Reference in New Issue
Block a user