diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000000..bbe2ee8e93 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,8 @@ +comment: false # Ref: https://docs.codecov.io/docs/pull-request-comments + +coverage: + status: + project: + default: + threshold: 0% # Ref: https://docs.codecov.io/docs/codecovyml-reference#coveragestatus + target: auto diff --git a/.eslintignore b/.eslintignore index dcf11353ec..46bb1ad2b2 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,7 +1,11 @@ **/node_modules/** **/dist/** +**/dist-types/** **/storybook-static/** **/coverage/** **/build/** **/.git/** **/public/** +**/microsite/** +**/templates/** +**/sample-templates/** diff --git a/.github/ISSUE_TEMPLATE/feature_template.md b/.github/ISSUE_TEMPLATE/feature_template.md index 012b8d7a06..d70622bf52 100644 --- a/.github/ISSUE_TEMPLATE/feature_template.md +++ b/.github/ISSUE_TEMPLATE/feature_template.md @@ -1,7 +1,7 @@ --- name: 'Feature Request' about: 'Suggest new features and changes' -labels: help wanted +labels: enhancement --- diff --git a/.github/ISSUE_TEMPLATE/rfc_template.md b/.github/ISSUE_TEMPLATE/rfc_template.md new file mode 100644 index 0000000000..c4990ee5d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc_template.md @@ -0,0 +1,26 @@ +--- +name: 'RFC' +about: 'Request For Comments (RFC) from the community' +labels: rfc +title: '[RFC] ' +--- + +**Status:** Open for comments + + + +## Need + + + +## Proposal + + + +## Alternatives + + + +## Risks + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 57746da2e8..0437698301 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,7 +4,9 @@ That makes it easier to understand the change so we can :shipit: faster. --> #### :heavy_check_mark: Checklist + + - [ ] All tests are passing `yarn test` - [ ] Screenshots attached (for UI changes) - [ ] Relevant documentation updated diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..59847fd65f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 5 + labels: + - dependencies + - package-ecosystem: npm + directory: '/microsite/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 2 + labels: + - dependencies diff --git a/.github/workflows/chromatic-storybook-test.yml b/.github/workflows/chromatic-storybook-test.yml index e38b494a73..0bcaf400d0 100644 --- a/.github/workflows/chromatic-storybook-test.yml +++ b/.github/workflows/chromatic-storybook-test.yml @@ -14,7 +14,37 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Required to retrieve git history - - run: yarn install && yarn build-storybook + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - run: yarn build-storybook + - uses: chromaui/action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..c8c92c57ff --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,116 @@ +name: CI + +on: + pull_request: + paths-ignore: + - 'microsite/**' +jobs: + verify: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + - name: fetch branch master + run: git fetch origin master + + # Beginning of yarn setup, keep in sync between all workflows. + # TODO(Rugvip): move this to composite action once all features we use are supported + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + + # Cache every node_modules folder inside the monorepo + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + # We use both yarn.lock and package.json as cache keys to ensure that + # changes to local monorepo packages bust the cache. + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + + # If we get a cache hit for node_modules, there's no need to bring in the global + # yarn cache or run yarn install, as all dependencies will be installed already. + + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: yarn install + if: steps.cache-modules.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + # End of yarn setup + + - name: check for yarn.lock changes + id: yarn-lock + run: git diff --quiet origin/master HEAD -- yarn.lock + continue-on-error: true + + - name: verify doc links + run: node docs/verify-links.js + + - name: prettier + run: yarn prettier:check + + - name: lint + run: yarn lerna -- run lint --since origin/master + + - name: type checking and declarations + run: yarn tsc:full + + - name: build changed packages + if: ${{ steps.yarn-lock.outcome == 'success' }} + run: yarn lerna -- run build --since origin/master + + - name: build all packages + if: ${{ steps.yarn-lock.outcome == 'failure' }} + run: yarn lerna -- run build + + - name: verify type dependencies + run: yarn lint:type-deps + + - name: test changed packages + if: ${{ steps.yarn-lock.outcome == 'success' }} + run: yarn lerna -- run test --since origin/master -- --coverage + + - name: test all packages (and upload coverage) + if: ${{ steps.yarn-lock.outcome == 'failure' }} + run: | + yarn lerna -- run test -- --coverage + bash <(curl -s https://codecov.io/bash) + + - name: verify plugin template + run: yarn lerna -- run diff -- --check + + - name: ensure clean working directory + run: | + if files=$(git ls-files --exclude-standard --others --modified) && [[ -z "$files" ]]; then + exit 0 + else + echo "" + echo "Working directory has been modified:" + echo "" + git status --short + echo "" + exit 1 + fi diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml deleted file mode 100644 index 7704aa36cd..0000000000 --- a/.github/workflows/cli.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: CLI Test - -on: - pull_request: - paths: - - '.github/workflows/cli.yml' - - 'packages/cli/**' - - 'packages/core/**' - - 'packages/core-api/**' - - 'yarn.lock' - -jobs: - build: - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu-latest] - node-version: [12.x] - - env: - CI: true - NODE_OPTIONS: --max-old-space-size=4096 - - name: Node ${{ matrix.node-version }} on ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: use node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: yarn install - run: yarn install --frozen-lockfile - - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation - run: | - sudo sysctl fs.inotify.max_user_watches=524288 - node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js diff --git a/.github/workflows/cli-win.yml b/.github/workflows/e2e-win.yml similarity index 54% rename from .github/workflows/cli-win.yml rename to .github/workflows/e2e-win.yml index 1355ed251e..5bab1b1299 100644 --- a/.github/workflows/cli-win.yml +++ b/.github/workflows/e2e-win.yml @@ -1,12 +1,14 @@ -name: CLI Test Windows +name: E2E Test Windows -# Building on windows is really slow, so this workflow is separate from cli.yml and only builds on changes +# Building on windows is really slow, so this workflow is separate from e2e.yml and only builds on changes # to the cli itself. They're more likely to introduce issues on windows, compared to changes to core and yarn.lock. on: pull_request: paths: - - '.github/workflows/cli-win.yml' + - '.github/workflows/e2e-win.yml' - 'packages/cli/**' + - 'packages/e2e/**' + - 'packages/create-app/**' jobs: build: @@ -24,23 +26,16 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - name: yarn install run: yarn install --frozen-lockfile + - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation - run: node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli + - name: run E2E test + run: yarn workspace e2e-test start diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..69bd8393c8 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,78 @@ +name: E2E Test Linux + +on: + pull_request: + paths-ignore: + - 'contrib/**' + - 'docs/**' + - 'microsite/**' + +jobs: + build: + runs-on: ${{ matrix.os }} + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432/tcp + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + matrix: + os: [ubuntu-latest] + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + name: Node ${{ matrix.node-version }} on ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - run: yarn tsc + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli + - name: run E2E test + run: | + sudo sysctl fs.inotify.max_user_watches=524288 + yarn workspace e2e-test start + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index f154c586fa..0000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Frontend CI - -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [12.x] - - env: - CI: true - NODE_OPTIONS: --max-old-space-size=4096 - - steps: - - uses: actions/checkout@v2 - - name: fetch branch master - run: git fetch origin master - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - - name: use node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - - name: check for yarn.lock changes - id: yarn-lock - run: git diff --quiet origin/master HEAD -- yarn.lock - continue-on-error: true - - - name: yarn install - run: yarn install --frozen-lockfile - - - name: lint - run: yarn lerna -- run lint --since origin/master - - - name: type checking and declarations - run: yarn tsc --incremental false - - - name: build changed packages - if: ${{ steps.yarn-lock.outcome == 'success' }} - # Need to build all dependencies as well to be able to run tests later - run: yarn lerna -- run build --since origin/master --include-dependencies - - - name: build all packages - if: ${{ steps.yarn-lock.outcome == 'failure' }} - run: yarn lerna -- run build - - - name: verify type dependencies - run: yarn lint:type-deps - - - name: test changed packages - if: ${{ steps.yarn-lock.outcome == 'success' }} - run: yarn lerna -- run test --since origin/master -- --coverage - - - name: test all packages - if: ${{ steps.yarn-lock.outcome == 'failure' }} - run: yarn lerna -- run test -- --coverage - - - name: verify plugin template - run: yarn lerna -- run diff -- --check - - - name: bundle example app - run: yarn bundle - - - name: verify storybook - run: yarn workspace storybook build-storybook diff --git a/.github/workflows/storybook-deploy.yml b/.github/workflows/master-win.yml similarity index 50% rename from .github/workflows/storybook-deploy.yml rename to .github/workflows/master-win.yml index 33ffba5ccd..3f65cf9aec 100644 --- a/.github/workflows/storybook-deploy.yml +++ b/.github/workflows/master-win.yml @@ -1,17 +1,12 @@ -name: Deploy Storybook +name: Master Build Windows on: push: - branches: - - master - paths: - - '.github/workflows/storybook-deploy.yml' - - 'packages/storybook/**' - - 'packages/core/src/**' + branches: [master] jobs: - deploy-storybook: - runs-on: ubuntu-latest + build: + runs-on: windows-latest strategy: matrix: @@ -23,6 +18,13 @@ jobs: steps: - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth - name: find location of global yarn cache id: yarn-cache run: echo "::set-output name=dir::$(yarn cache dir)" @@ -30,27 +32,29 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - - - name: use node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - registry-url: https://registry.npmjs.org/ # Needed for auth - name: yarn install run: yarn install --frozen-lockfile - - name: build storybook - run: yarn workspace storybook build-storybook - - name: deploy storybook to gh-pages - uses: JamesIves/github-pages-deploy-action@3.4.2 + # End of yarn setup + + - name: lint + run: yarn lerna -- run lint + + - name: type checking and declarations + run: yarn tsc:full + + - name: verify type dependencies + run: yarn lint:type-deps + + - name: test + run: yarn lerna -- run test + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: packages/storybook/dist + args: 'Windows master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3ef0c1fee3..f99c266241 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1,4 +1,4 @@ -name: Master Build +name: Main Master Build on: push: @@ -18,35 +18,40 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup - name: lint run: yarn lerna -- run lint - name: type checking and declarations - run: yarn tsc --incremental false + run: yarn tsc:full - name: build run: yarn build @@ -54,8 +59,10 @@ jobs: - name: verify type dependencies run: yarn lint:type-deps - - name: test - run: yarn lerna -- run test -- --coverage + - name: test (and upload coverage) + run: | + yarn lerna -- run test -- --coverage + bash <(curl -s https://codecov.io/bash) # Publishes current version of packages that are not already present in the registry - name: publish @@ -66,6 +73,14 @@ jobs: # Tags the commit with the version in the core package if the tag doesn't exist - uses: Klemensas/action-autotag@1.2.3 with: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - package_root: "packages/core" - tag_prefix: "v" + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + package_root: 'packages/core' + tag_prefix: 'v' + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/microsite-build-check.yml b/.github/workflows/microsite-build-check.yml new file mode 100644 index 0000000000..ff290b71e4 --- /dev/null +++ b/.github/workflows/microsite-build-check.yml @@ -0,0 +1,38 @@ +name: Build microsite + +on: + pull_request: + paths: + - '.github/workflows/microsite-build-check.yml' + - 'microsite/**' + - 'docs/**' + +jobs: + build-microsite: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + # Skip caching of microsite dependencies, it keeps the global cache size + # smaller, which make Windows builds a lot faster for the rest of the project. + - name: yarn install + run: yarn install --frozen-lockfile + working-directory: microsite + + - name: build microsite + run: yarn build + working-directory: microsite diff --git a/.github/workflows/microsite-with-storybook-deploy.yml b/.github/workflows/microsite-with-storybook-deploy.yml new file mode 100644 index 0000000000..22fd48789a --- /dev/null +++ b/.github/workflows/microsite-with-storybook-deploy.yml @@ -0,0 +1,62 @@ +name: Deploy Microsite and Storybook + +on: + push: + branches: + - master + paths: + - '.github/workflows/microsite-with-storybook-deploy.yml' + - 'packages/storybook/**' + - 'packages/core/src/**' + - 'microsite/**' + - 'docs/**' + +jobs: + deploy-microsite-and-storybook: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + + # We avoid caching in this workflow, as we're running an install of both the top-level + # dependencies and the microsite. We leave it to the main master workflow to produce the + # cache, as that results in a smaller bundle. + - name: top-level yarn install + run: yarn install --frozen-lockfile + - name: microsite yarn install + run: yarn install --frozen-lockfile + working-directory: microsite + + - name: build microsite + run: yarn build + working-directory: microsite + + - name: build storybook + run: yarn workspace storybook build-storybook + + - name: move storybook dist into microsite + run: mv packages/storybook/dist/ microsite/build/backstage/storybook + + - name: Check the build output + run: ls microsite/build/backstage && ls microsite/build/backstage/storybook + + - name: Deploy both microsite and storybook to gh-pages + uses: JamesIves/github-pages-deploy-action@3.4.2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: microsite/build/backstage diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml new file mode 100644 index 0000000000..b389cf6bf5 --- /dev/null +++ b/.github/workflows/techdocs-project-board.yml @@ -0,0 +1,53 @@ +name: Automatically add new TechDocs Issues and PRs to the GitHub project board +# Development of TechDocs in Backstage is managed by this Kanban board - https://github.com/spotify/backstage/projects/5 +# New issues with TechDocs in their title or docs-like-code label will be added to the board. + +on: + issues: + types: [opened, reopened, labeled, edited] + pull_request: + types: [opened, reopened, labeled, edited] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + assign_issue_or_pr_to_project: + runs-on: ubuntu-latest + name: Triage + steps: + - name: Assign new issue to Incoming based on its title. + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + contains(github.event.issue.title, 'TechDocs') || + contains(github.event.issue.title, 'techdocs') || + contains(github.event.issue.title, 'Techdocs') + with: + project: 'https://github.com/spotify/backstage/projects/5' + column_name: 'Incoming' + + - name: Assign new issue to Incoming based on its label. + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + contains(github.event.issue.labels.*.name, 'docs-like-code') + with: + project: 'https://github.com/spotify/backstage/projects/5' + column_name: 'Incoming' + + - name: Assign new PR to Incoming based on its title. + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + contains(github.event.pull_request.title, 'TechDocs') || + contains(github.event.pull_request.title, 'techdocs') || + contains(github.event.pull_request.title, 'Techdocs') + with: + project: 'https://github.com/spotify/backstage/projects/5' + column_name: 'Incoming' + + - name: Assign new PR to Incoming based on its label. + uses: srggrs/assign-one-project-github-action@1.2.0 + if: | + contains(github.event.pull_request.labels.*.name, 'docs-like-code') + with: + project: 'https://github.com/spotify/backstage/projects/5' + column_name: 'Incoming' diff --git a/.gitignore b/.gitignore index c1bcf5a4b4..e7d294d55a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,12 +3,6 @@ .vscode/ .vsls.json -# @spotify/web-script build output -cjs/ -esm/ -types/ -build/ - # Logs logs *.log @@ -95,6 +89,7 @@ typings/ # Nuxt.js build / generate output .nuxt dist +dist-types # Gatsby files .cache/ @@ -125,3 +120,6 @@ dist # MkDocs build output site + +# Local configuration files +*.local.yaml diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..9e75b74eee --- /dev/null +++ b/.prettierignore @@ -0,0 +1,8 @@ +.yarn +dist +microsite/build +coverage +*.hbs +templates +plugins/scaffolder-backend/sample-templates +.vscode diff --git a/ADOPTERS.md b/ADOPTERS.md index 63aae59401..0ba3e3196b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,12 +1,15 @@ -| Organization | Contact | Description of Use | -| ---------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| Organization | Contact | Description of Use | +| -------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..cf68fad6bd --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,117 @@ +# Backstage Changelog + +This is a best-effort changelog where we manually collect breaking changes. It is not an exhaustive list of all changes or even features added. + +If you encounter issues while upgrading to a newer version, don't hesitate to reach out on [Discord](https://discord.gg/EBHEGzX) or [open an issue](https://github.com/spotify/backstage/issues/new/choose)! + +## Next Release + +> Collect changes for the next release below + +## v0.1.1-alpha.22 + +### @backstage/core + +- Introduced initial version of an inverted app/plugin relationship, where plugins export components for apps to use, instead registering themselves directly into the app. This enables more fine-grained control of plugin features, and also composition of plugins such as catalog pages with additional cards and tabs. This breaks the use of `RouteRef`s, and there will be more changes related to this in the future, but this change lays the initial foundation. See `packages/app` and followup PRs for how to update plugins for this change. [#2076](https://github.com/spotify/backstage/pull/2076) +- Switch to an automatic dependency injection mechanism for all Utility APIs, allowing plugins to ship default implementations of their APIs. See [https://backstage.io/docs/api/utility-apis](https://backstage.io/docs/api/utility-apis). [#2285](https://github.com/spotify/backstage/pull/2285) + +### @backstage/cli + +- Change `backstage-cli backend:build-image` to forward all args to `docker image build`, instead of just tag. Also add `--build` flag for building all dependent packages before packaging the workspace for the docker build. [#2299](https://github.com/spotify/backstage/pull/2299) + +### @backstage/create-app + +- Change root `tsc` output dir to `dist-types`, in order to allow for standalone plugin repos. [#2278](https://github.com/spotify/backstage/pull/2278) + +### @backstage/catalog-backend + +- We have simplified the way that GitHub ingestion works. The `catalog.processors.githubApi` key is deprecated, in favor of `catalog.processors.github`. At the same time, the location type `github/api` is likewise deprecated, in favor of `github`. This location type now serves both raw HTTP reads and APIv3 reads, depending on how you configure it. It also supports having several providers at once - for example, both public GitHub and an internal GitHub Enterprise, with different keys. If you still use the `catalog.processors.githubApi` config key, things will work but you will get a deprecation warning at startup. In a later release, support for the old key will go away entirely. See the [configuration section in the docs](https://backstage.io/docs/features/software-catalog/configuration) for more details. + +## v0.1.1-alpha.21 + +- Added many more frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084) + +### @backstage/core + +- Material-UI: Bumped to 4.11.0, which is the version that create-app will + resolve to, because we wanted to get the renaming of ExpansionPanel to + Accordion into place. This gets rid of a lot of console deprecation warnings + in newly scaffolded apps. + +### @backstage/cli + +- Set `NODE_ENV` to `test` when running test. [#2214](https://github.com/spotify/backstage/pull/2214) + +- Fix for backend plugins names requiring to be prefixed with `@backstage` to build. [#2224](https://github.com/spotify/backstage/pull/2224) + +### @backstage/backend-common + +- The backend plugin + [service builder](https://github.com/spotify/backstage/blob/master/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts) + no longer adds `express.json()` automatically to all routes. While convenient + in a lot of cases, it also led to problems where for example the proxy + middleware could hang because the body had already been altered and could not + be streamed. Also, plugins that rather wanted to handle e.g. form encoded data + still had to cater to that manually. We therefore decided to let plugins add + `express.json()` themselves if they happen to deal with JSON data. + +### @backstage/catalog-backend + +- Add rules configuration for catalog location and entity kinds. The default rules should cover most use-cases, but you may need to allow specific entity kinds when using things like Template or Group entities. [#2118](https://github.com/spotify/backstage/pull/2118) + +## v0.1.1-alpha.20 + +### @backstage/cli + +- Use config files according to `NODE_ENV` when serving and building frontend packages. [#2077](https://github.com/spotify/backstage/pull/2077) + +- Pin `rollup-plugin-dts` to avoid a later broken version. [#2097](https://github.com/spotify/backstage/pull/2097) + +## v0.1.1-alpha.19 + +### @backstage/backend-common + +- Allow listen host and port to be configured separately, in order to support PORT environment variables. [#1950](https://github.com/spotify/backstage/pull/1950) + +### @backstage/core + +- Added new `DiscoveryApi` for discovering backend endpoint in the frontend, and use in most plugins. See [packages/app/src/apis.ts](https://github.com/spotify/backstage/blob/master/packages/app/src/apis.ts) for how to register in your app. [#2074](https://github.com/spotify/backstage/pull/2074) + +### @backstage/create-app + +- Added catalog and scaffolder frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084) +- Many plugins have been added to the catalog and will for now be required to be added to separate apps as well. This will be solved as [#1536](https://github.com/spotify/backstage/issues/1536) gets sorted out, but for now you may need to install some plugins just to get pages to work. + +### @backstage/catalog-backend + +- Added the possibility to add static locations via `app-config.yaml`. This changed the signature of `new LocationReaders(logger)` inside `packages/backend/src/plugins/catalog.ts` to `new LocationReaders({config, logger})`. [#1890](https://github.com/spotify/backstage/pull/1890) + +### @backstage/theme + +- Changed the type signature of the palette, removing `sidebar: string` and adding `navigation: { background: string; indicator: string}`. [#1880](https://github.com/spotify/backstage/pull/1880) + +## v0.1.1-alpha.18 + +### @backstage/catalog-backend + +- Fixed an issue with duplicated location logs. Applying the database migrations from this fix will clear the existing migration logs. [#1836](https://github.com/spotify/backstage/pull/1836) + +### @backstage/auth-backend + +This version fixes a breakage in CSP policies set by the auth backend. If you're facing trouble with auth in alpha.17, upgrade to alpha.18. + +- OAuth redirect URLs no longer receive the `env` parameter, as it is now passed through state instead. This will likely require a reconfiguration of the OAuth app, where a redirect URL like `http://localhost:7000/auth/google/handler/frame?env=development` should now be configured as `http://localhost:7000/auth/google/handler/frame`. [#1812](https://github.com/spotify/backstage/pull/1812) + +### @backstage/core + +- `SignInPage` props have been changed to receive a list of provider objects instead of simple string identifiers for all but the `'guest'` and `'custom'` providers. This opens up for configuration of custom providers, but may break existing configurations. See [packages/app/src/App.tsx](https://github.com/spotify/backstage/blob/032ba401af36a760efdac41668d7000ccf09bc57/packages/app/src/App.tsx#L36) and [packages/app/src/identityProviders.ts](https://github.com/spotify/backstage/blob/032ba401af36a760efdac41668d7000ccf09bc57/packages/app/src/identityProviders.ts#L24) for how to bring back the existing providers. [#1816](https://github.com/spotify/backstage/pull/1816) + +## v0.1.1-alpha.17 + +### @backstage/techdocs-backend + +- The techdocs backend now requires more configuration to be supplied when creating the router. See [packages/backend/src/plugins/techdocs.ts](https://github.com/spotify/backstage/blob/0201fd9b4a52429519dd59e9184106ba69456deb/packages/backend/src/plugins/techdocs.ts#L42) for an example. [#1736](https://github.com/spotify/backstage/pull/1736) + +### @backstage/cli + +- The `create-app` command was moved out from the CLI to a standalone package. It's now invoked with `npx @backstage/create-app` instead. [#1745](https://github.com/spotify/backstage/pull/1745) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 69f6457dd9..55269dd2a5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -4,12 +4,12 @@ This code of conduct outlines our expectations for participants within the **Spo Our open source community strives to: -* **Be friendly and patient.** -* **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability. -* **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language. -* **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one. -* **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable. -* **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes. +- **Be friendly and patient.** +- **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability. +- **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language. +- **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one. +- **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable. +- **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes. ## Definitions @@ -18,7 +18,7 @@ Harassment includes, but is not limited to: - Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation - Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment - Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle -- Physical contact and simulated physical contact (eg, textual descriptions like “*hug*” or “*backrub*”) without consent or after a request to stop +- Physical contact and simulated physical contact (eg, textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop - Threats of violence, both physical and psychological - Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm - Deliberate intimidation @@ -39,7 +39,6 @@ Our open source community prioritizes marginalized people’s safety over privil - Communicating in a ‘tone’ you don’t find congenial - Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions - ### Diversity Statement We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong. @@ -53,18 +52,18 @@ If you experience or witness unacceptable behavior—or have any other concerns - Your contact information. - Names (real, nicknames, or pseudonyms) of any individuals involved. If there are additional witnesses, please -include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link. + include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link. - Any additional information that may be helpful. After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse. ### Attribution & Acknowledgements -We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration: +We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration: -* [Django](https://www.djangoproject.com/conduct/reporting/) -* [Python](https://www.python.org/community/diversity/) -* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct) -* [Contributor Covenant](http://contributor-covenant.org/) -* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/) -* [Citizen Code of Conduct](http://citizencodeofconduct.org/) +- [Django](https://www.djangoproject.com/conduct/reporting/) +- [Python](https://www.python.org/community/diversity/) +- [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct) +- [Contributor Covenant](http://contributor-covenant.org/) +- [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/) +- [Citizen Code of Conduct](http://citizencodeofconduct.org/) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6200f3235b..0d6a8db50c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing +# Contributing to Backstage Our vision for Backstage is for it to become the trusted standard toolbox (read: UX layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. @@ -28,7 +28,7 @@ What kind of plugins should/could be created? Some inspiration from the 120+ plu ## Suggesting a plugin -If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?template=plugin_template.md). This helps the community know what plugins are in development. +If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME). This helps the community know what plugins are in development. You can also use this process if you have an idea for a good plugin but you hope that someone else will pick up the work. @@ -38,7 +38,7 @@ The current documentation is very limited. Help us make the `/docs` folder come ## Contribute to Storybook -We think the best way to ensure different plugins provide a consistent experience is through a solid set of reusable UI/UX components. Backstage uses [Storybook](http://storybook.backstage.io). +We think the best way to ensure different plugins provide a consistent experience is through a solid set of reusable UI/UX components. Backstage uses [Storybook](http://backstage.io/storybook). Either help us [create new components](https://github.com/spotify/backstage/labels/help%20wanted) or improve stories for the existing ones (look for files with `*.stories.tsx`). @@ -60,9 +60,17 @@ Have you started using Backstage? Adding your company to [ADOPTERS](ADOPTERS.md) # Get Started! -So...feel ready to jump in? Let's do this. Head over to the [Getting Started guide](https://github.com/spotify/backstage#getting-started) 👏🏻💯 +So...feel ready to jump in? Let's do this. 👏🏻💯 -If you need help, just jump into our [Discord chatroom](https://discord.gg/MUpMjP2). +Start by reading our [Getting Started](https://backstage.io/docs/getting-started/) page. If you need help, just jump into our [Discord chatroom](https://discord.gg/MUpMjP2). + +# Coding Guidelines + +All code is formatted with `prettier` using the configuration in the repo. If possible we recommend configuring your editor to format automatically, but you can also use the `yarn prettier --write ` command to format files. + +If you're contributing to the backend or CLI tooling, be mindful of cross-platform support. [This](https://shapeshed.com/writing-cross-platform-node/) blog post is a good guide of what to keep in mind when writing cross-platform NodeJS. + +Also be sure to skim through our [ADRs](https://github.com/spotify/backstage/tree/master/docs/architecture-decisions) to see if they cover what you're working on. In particular [ADR006: Avoid React.FC and React.SFC](https://github.com/spotify/backstage/blob/master/docs/architecture-decisions/adr006-avoid-react-fc.md) is one to look out for. # Code of Conduct diff --git a/Dockerfile b/Dockerfile index fa89debcee..174548a90c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,14 @@ FROM nginx:mainline +# The purpose of this image is to serve the frontend app content separately. +# By default the Backstage backend uses the app-backend plugin to serve the +# app from the backend itself, but it may be desirable to move the frontend +# content serving to a separate deployment, in which case this image can be used. + # This dockerfile requires the app to be built on the host first, as it # simply copies in the build output into the image. -# The safest way to build this image is to use `yarn docker-build` +# The safest way to build this image is to use `yarn docker-build:app` RUN apt-get update && apt-get -y install jq && rm -rf /var/lib/apt/lists/* diff --git a/OWNERS.md b/OWNERS.md new file mode 100644 index 0000000000..1e7fe5b774 --- /dev/null +++ b/OWNERS.md @@ -0,0 +1,19 @@ +# Owners + +- See [CONTRIBUTING.md](CONTRIBUTING.md) for general contribution guidelines. + +## Maintainers 🏓 + +- Patrik Oldsberg (@Rugvip, Spotify) +- Fredrik Adelöw (@freben, Spotify) +- Raghunandan Balachandran (@soapraj, Spotify) +- Ben Lambert (@benjdlambert, Spotify) +- Marcus Eide (@marcuseide, Spotify) +- Niklas Ek (@nikek, Spotify) +- Stefan Ålund (@stefanalund, Spotify) +- Kat Zhou (@katz95, Spotify) + +## Hall of Fame 👏 + +- Andrew Thauer (@andrewthauer, Wealthsimple) +- Oliver Sand (@Fox32, SDA-SE) diff --git a/README.md b/README.md index 8e7574e128..9ef09aee06 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![headline](docs/headline.png) +![headline](docs/assets/headline.png) # [Backstage](https://backstage.io) @@ -6,105 +6,48 @@ ![](https://github.com/spotify/backstage/workflows/Frontend%20CI/badge.svg) [![Discord](https://img.shields.io/discord/687207715902193673)](https://discord.gg/EBHEGzX) ![Code style](https://img.shields.io/badge/code_style-prettier-ff69b4.svg) +[![Codecov](https://img.shields.io/codecov/c/github/spotify/backstage)](https://codecov.io/gh/spotify/backstage) [![](https://img.shields.io/npm/v/@backstage/core?label=Version)](https://github.com/spotify/backstage/releases) ## What is Backstage? -[Backstage](https://backstage.io/) is an open platform for building developer portals. It’s based on the developer portal we’ve been using internally at Spotify for over four years. Backstage can be as simple as a services catalog or as powerful as the UX layer for your entire tech infrastructure. +[Backstage](https://backstage.io/) is an open platform for building developer portals. Powered by a centralized service catalog, Backstage restores order to your microservices and infrastructure. So your product teams can ship high-quality code quickly — without compromising autonomy. -For more information go to [backstage.io](https://backstage.io) or join our [Discord chatroom](https://discord.gg/EBHEGzX). - -### Features - -- Create and manage all of your organization’s software and microservices in one place. -- Services catalog keeps track of all software and its ownership. -- Visualizations provide information about your backend services and tooling, and help you monitor them. -- A unified method for managing microservices offers both visibility and control. -- Preset templates allow engineers to quickly create microservices in a standardized way ([coming soon](https://github.com/spotify/backstage/milestone/11)). -- Centralized, full-featured technical documentation with integrated tooling that makes it easy for developers to set up, publish, and maintain alongside their code ([coming soon](https://github.com/spotify/backstage/milestone/15)). - -### Benefits - -- For _engineering managers_, it allows you to maintain standards and best practices across the organization, and can help you manage your whole tech ecosystem, from migrations to test certification. -- For _end users_ (developers), it makes it fast and simple to build software components in a standardized way, and it provides a central place to manage all projects and documentation. -- For _platform engineers_, it enables extensibility and scalability by letting you easily integrate new tools and services (via plugins), as well as extending the functionality of existing ones. -- For _everyone_, it’s a single, consistent experience that ties all your infrastructure tooling, resources, standards, owners, contributors, and administrators together in one place. - -## Backstage Service Catalog (alpha) - -The Backstage Service Catalog — actually, a software catalog, since it includes more than just services — is a centralized system that keeps track of ownership and metadata for all the software in your ecosystem (services, websites, libraries, data pipelines, etc). The catalog is built around the concept of [metadata yaml files](https://github.com/spotify/backstage/blob/master/docs/architecture-decisions/adr002-default-catalog-file-format.md#format) stored together with the code, which are then harvested and visualized in Backstage. +Backstage unifies all your infrastructure tooling, services, and documentation to create a streamlined development environment from end to end. ![service-catalog](https://backstage.io/blog/assets/6/header.png) -We have also found that the service catalog is a great way to organise the infrastructure tools you use to manage the software as well. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different infrastructure UI’s (and incurring additional cognitive overhead each time they make a context switch), most of these tools can be organised around the entities in the catalog. +Out of the box, Backstage includes: + +- [Backstage Service Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview) for managing all your software (microservices, libraries, data pipelines, websites, ML models, etc.) +- [Backstage Software Templates](https://backstage.io/docs/features/software-templates/software-templates-index) for quickly spinning up new projects and standardizing your tooling with your organization’s best practices +- [Backstage TechDocs](https://backstage.io/docs/features/techdocs/techdocs-overview) for making it easy to create, maintain, find, and use technical documentation, using a "docs like code" approach +- Plus, a growing ecosystem of [open source plugins](https://github.com/spotify/backstage/tree/master/plugins) that further expand Backstage’s customizability and functionality + +For more information go to [backstage.io](https://backstage.io) or join our [Discord chatroom](https://discord.gg/EBHEGzX). ## Project roadmap -We created Backstage about 4 years ago. While our internal version of Backstage has had the benefit of time to mature and evolve, the first iteration of our open source version is still nascent. We are envisioning three phases of the project and we have already begun work on various aspects of these phases: +A detailed project roadmap, including already delivered milestones, is available [here](https://backstage.io/docs/overview/roadmap). -- 🐣 **Phase 1:** Extensible frontend platform (Done ✅) - You will be able to easily create a single consistent UI layer for your internal infrastructure and tools. A set of reusable [UX patterns and components](http://storybook.backstage.io) help ensure a consistent experience between tools. +## Getting Started -- 🐢 **Phase 2:** Service Catalog ([alpha released](https://backstage.io/blog/2020/06/22/backstage-service-catalog-alpha)) - With a single catalog, Backstage makes it easy for a team to manage ten services — and makes it possible for your company to manage thousands of them. Developers can get a uniform overview of all their software and related resources, regardless of how and where they are running, as well as an easy way to onboard and manage those resources. - -- 🐇 **Phase 3:** Ecosystem (later) - Everyone's infrastructure stack is different. By fostering a vibrant community of contributors we hope to provide an ecosystem of Open Source plugins/integrations that allows you to pick the tools that match your stack. - -Check out our [Milestones](https://github.com/spotify/backstage/milestones) and open [RFCs](https://github.com/spotify/backstage/labels/rfc) how they relate to the three Phases outlined above. - -Our vision for Backstage is for it to become the trusted standard toolbox (read: UX layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. If this sounds interesting or you'd like to help us shape our product vision, we'd love to talk. You can email me directly: [alund@spotify.com](mailto:alund@spotify.com). - -## Overview - -The Backstage platform consists of a number of different components: - -- **app** - Main web application that users interact with. It's built up by a number of different _Plugins_. This repo contains an example implementation of an app (located in `packages/app`) and you can easily get started with your own app by [creating one](docs/create-an-app.md). -- [**plugins**](https://github.com/spotify/backstage/tree/master/plugins) - Each plugin is treated as a self-contained web app and can include almost any type of content. Plugins all use a common set of platform API's and reusable UI components. Plugins can fetch data either from the _backend_ or through any RESTful API exposed through the _proxy_. -- [**service catalog**](https://github.com/spotify/backstage/tree/master/packages/backend) - Service that holds the model of your software ecosystem, including organisational information and what team owns what software. The backend also has a Plugin model for extending its graph. -- [**proxy**](https://github.com/spotify/backstage/tree/master/plugins/proxy-backend) - Terminates HTTPS and exposes any RESTful API to Plugins. -- **identity** - A backend service that holds your organisation's metadata. - -## Getting started - -To run a Backstage app, you will need to have the following installed: - -- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -- [NodeJS](https://nodejs.org/en/download/) - Active LTS Release, currently v12 -- [yarn](https://classic.yarnpkg.com/en/docs/install) - -After cloning this repo, open a terminal window and start the example app using the following commands from the project root: - -```bash -yarn install # Install dependencies - -yarn start # Start dev server, use --check to enable linting and type-checks -``` - -The final `yarn start` command should open a local instance of Backstage in your browser, otherwise open one of the URLs printed in the terminal. - -And that's it! You are good to go 👍 - -### Next step - -Take a look at the [Getting Started](docs/getting-started/index.md) guide to learn how to set up Backstage, and how to develop on the platform. +Check out [the documentation](https://backstage.io/docs/getting-started) on how to start using Backstage. ## Documentation -- [Main documentation](docs/README.md) -- [Service Catalog](docs/features/software-catalog/index.md) -- [Create a Backstage App](docs/getting-started/create-an-app.md) -- [Architecture](docs/overview/architecture-terminology.md) ([Decisions](docs/architecture-decisions/index.md)) -- [Designing for Backstage](docs/dls/design.md) -- [Storybook - UI components](http://storybook.backstage.io) - -## Contributing - -We would love your help in building Backstage! See [CONTRIBUTING](CONTRIBUTING.md) for more information. +- [Main documentation](https://backstage.io/docs) +- [Service Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview) +- [Architecture](https://backstage.io/docs/overview/architecture-terminology) ([Decisions](https://backstage.io/docs/architecture-decisions/adrs-overview)) +- [Designing for Backstage](https://backstage.io/docs/dls/design) +- [Storybook - UI components](https://backstage.io/storybook) ## Community - [Discord chatroom](https://discord.gg/MUpMjP2) - Get support or discuss the project - [Good First Issues](https://github.com/spotify/backstage/contribute) - Start here if you want to contribute - [RFCs](https://github.com/spotify/backstage/labels/rfc) - Help shape the technical direction -- [FAQ](docs/FAQ.md) - Frequently Asked Questions +- [FAQ](https://backstage.io/docs/FAQ) - Frequently Asked Questions - [Code of Conduct](CODE_OF_CONDUCT.md) - This is how we roll - [Adopters](ADOPTERS.md) - Companies already using Backstage - [Blog](https://backstage.io/blog/) - Announcements and updates diff --git a/app-config.development.yaml b/app-config.development.yaml new file mode 100644 index 0000000000..da274ba1a8 --- /dev/null +++ b/app-config.development.yaml @@ -0,0 +1,11 @@ +app: + baseUrl: http://localhost:3000 + +backend: + baseUrl: http://localhost:7000 + listen: + port: 7000 + cors: + origin: http://localhost:3000 + methods: [GET, POST, PUT, DELETE] + credentials: true diff --git a/app-config.yaml b/app-config.yaml index 9a4a14b923..60ecb813f7 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -1,27 +1,211 @@ app: title: Backstage Example App - baseUrl: http://localhost:3000 + baseUrl: http://localhost:7000 backend: baseUrl: http://localhost:7000 - listen: 0.0.0.0:7000 - cors: - origin: http://localhost:3000 - methods: [GET, POST, PUT, DELETE] - credentials: true + listen: + port: 7000 + database: + client: sqlite3 + connection: ':memory:' +# See README.md in the proxy-backend plugin for information on the configuration format proxy: - "/circleci/api": - target: "https://circleci.com/api/v1.1" + '/circleci/api': + target: https://circleci.com/api/v1.1 changeOrigin: true pathRewrite: - "^/proxy/circleci/api/": "/" + '^/proxy/circleci/api/': '/' + headers: + Circle-Token: + $secret: + env: CIRCLECI_AUTH_TOKEN + + '/jenkins/api': + target: http://localhost:8080 + headers: + Authorization: + $secret: + env: JENKINS_BASIC_AUTH_HEADER organization: name: Spotify techdocs: - storageUrl: https://techdocs-mock-sites.storage.googleapis.com + storageUrl: http://localhost:7000/techdocs/static/docs + requestUrl: http://localhost:7000/techdocs/docs + generators: + techdocs: 'docker' sentry: organization: spotify + +rollbar: + organization: spotify + accountToken: + $secret: + env: ROLLBAR_ACCOUNT_TOKEN + +newrelic: + api: + baseUrl: 'https://api.newrelic.com/v2' + key: NEW_RELIC_REST_API_KEY + +lighthouse: + baseUrl: http://localhost:3003 + +catalog: + rules: + - allow: [Component, API, Group, Template, Location] + processors: + github: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using the API: + # - target: https://ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - target: https://ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + bitbucketApi: + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + gitlabApi: + privateToken: + $secret: + env: GITLAB_PRIVATE_TOKEN + azureApi: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN + + locations: + # Backstage example components + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml + + # Example component for github-actions + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml + + # Example component for techdocs + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml + + # Backstage example APIs + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml + + # Backstage example templates + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml + +scaffolder: + github: + token: + $secret: + env: GITHUB_ACCESS_TOKEN + visibility: public # or 'internal' or 'private' + gitlab: + api: + baseUrl: https://gitlab.com + token: + $secret: + env: GITLAB_ACCESS_TOKEN + +auth: + providers: + google: + development: + clientId: + $secret: + env: AUTH_GOOGLE_CLIENT_ID + clientSecret: + $secret: + env: AUTH_GOOGLE_CLIENT_SECRET + github: + development: + clientId: + $secret: + env: AUTH_GITHUB_CLIENT_ID + clientSecret: + $secret: + env: AUTH_GITHUB_CLIENT_SECRET + enterpriseInstanceUrl: + $secret: + env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL + gitlab: + development: + clientId: + $secret: + env: AUTH_GITLAB_CLIENT_ID + clientSecret: + $secret: + env: AUTH_GITLAB_CLIENT_SECRET + audience: + $secret: + env: GITLAB_BASE_URL + saml: + entryPoint: 'http://localhost:7001/' + issuer: 'passport-saml' + okta: + development: + clientId: + $secret: + env: AUTH_OKTA_CLIENT_ID + clientSecret: + $secret: + env: AUTH_OKTA_CLIENT_SECRET + audience: + $secret: + env: AUTH_OKTA_AUDIENCE + oauth2: + development: + clientId: + $secret: + env: AUTH_OAUTH2_CLIENT_ID + clientSecret: + $secret: + env: AUTH_OAUTH2_CLIENT_SECRET + authorizationUrl: + $secret: + env: AUTH_OAUTH2_AUTH_URL + tokenUrl: + $secret: + env: AUTH_OAUTH2_TOKEN_URL + auth0: + development: + clientId: + $secret: + env: AUTH_AUTH0_CLIENT_ID + clientSecret: + $secret: + env: AUTH_AUTH0_CLIENT_SECRET + domain: + $secret: + env: AUTH_AUTH0_DOMAIN + microsoft: + development: + clientId: + $secret: + env: AUTH_MICROSOFT_CLIENT_ID + clientSecret: + $secret: + env: AUTH_MICROSOFT_CLIENT_SECRET + tenantId: + $secret: + env: AUTH_MICROSOFT_TENANT_ID diff --git a/catalog-info.yaml b/catalog-info.yaml new file mode 100644 index 0000000000..937a55bfca --- /dev/null +++ b/catalog-info.yaml @@ -0,0 +1,13 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage + description: | + Backstage is an open-source developer portal that puts the developer experience first. + annotations: + github.com/project-slug: spotify/backstage + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git +spec: + type: library + owner: Spotify + lifecycle: experimental diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..2bf33c063b --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Backstage Contrib + +This directory contains various community contributions related to Backstage. + +Unless otherwise specified, all content in this hierarchy fall under the same +[licensing terms](../LICENSE) as in the rest of the repository, and come with +no guarantees of functionality or fitness of purpose. That being said, we +really appreciate contributions in here and encourage them being kept up to +date. diff --git a/contrib/docker/multi-stage-frontend/Dockerfile b/contrib/docker/multi-stage-frontend/Dockerfile new file mode 100644 index 0000000000..0c492478d1 --- /dev/null +++ b/contrib/docker/multi-stage-frontend/Dockerfile @@ -0,0 +1,23 @@ +FROM node:12 AS build + +RUN mkdir /app +COPY . /app +WORKDIR /app + +RUN yarn install +RUN yarn workspace example-app build + +# Contruct backstage-frontend image +FROM nginx:mainline + +RUN apt-get update && apt-get -y install jq && rm -rf /var/lib/apt/lists/* + +# Copy from build stage +COPY --from=build /app/packages/app/dist /usr/share/nginx/html + +COPY docker/default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker/run.sh /usr/local/bin/run.sh + +CMD run.sh + +ENV PORT 80 diff --git a/contrib/docker/multi-stage-frontend/README.md b/contrib/docker/multi-stage-frontend/README.md new file mode 100644 index 0000000000..b6ae4fdc6c --- /dev/null +++ b/contrib/docker/multi-stage-frontend/README.md @@ -0,0 +1,19 @@ +# Standalone Dockerfile for frontend + +This directory contains the resources which will help you build backstage without any requirements +other than docker itself. It uses a multi-stage Dockerfile to build and ship backstage. + +## Usage + +You can simply run the following command to build backstage. + +``` +# Make sure you are in the root directory of backstage then run +docker build -t backstage-frontend -f ./contrib/docker/multi-stage-frontend/Dockerfile . +``` + +After a successful build, You can simply run backstage frontend with the following command. + +``` +docker run -it --rm -p 3080:80 backstage-frontend +``` diff --git a/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md b/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md new file mode 100644 index 0000000000..2dc875f671 --- /dev/null +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md @@ -0,0 +1 @@ +# Basic Kubernetes example with Helm diff --git a/install/kubernetes/app.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/app.yaml similarity index 54% rename from install/kubernetes/app.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/app.yaml index ed4108f11b..c54e25d519 100644 --- a/install/kubernetes/app.yaml +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/app.yaml @@ -11,17 +11,17 @@ spec: matchLabels: app: backstage component: frontend - template: - metadata: - labels: - app: backstage - component: frontend - spec: - containers: + template: + metadata: + labels: + app: backstage + component: frontend + spec: + containers: - name: app image: spotify/backstage:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent ports: - - containerPort: 80 - name: app - protocol: TCP + - containerPort: 80 + name: app + protocol: TCP diff --git a/install/kubernetes/backend.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backend.yaml similarity index 55% rename from install/kubernetes/backend.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backend.yaml index 669426fecd..f0695753fd 100644 --- a/install/kubernetes/backend.yaml +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/backend.yaml @@ -11,17 +11,17 @@ spec: matchLabels: app: backstage component: backend - template: - metadata: - labels: - app: backstage - component: backend - spec: - containers: + template: + metadata: + labels: + app: backstage + component: backend + spec: + containers: - name: backend image: spotify/backstage-backend:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent ports: - - containerPort: 7000 - name: backend - protocol: TCP + - containerPort: 7000 + name: backend + protocol: TCP diff --git a/install/kubernetes/backstage/.helmignore b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/.helmignore similarity index 100% rename from install/kubernetes/backstage/.helmignore rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/.helmignore diff --git a/install/kubernetes/backstage/Chart.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/Chart.yaml similarity index 85% rename from install/kubernetes/backstage/Chart.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/Chart.yaml index 23ea41ec41..efc3635a7f 100644 --- a/install/kubernetes/backstage/Chart.yaml +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.0" +appVersion: '1.0' description: A Helm chart for Spotify Backstage name: backstage version: 0.1.1-alpha.12 diff --git a/install/kubernetes/backstage/README.md b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/README.md similarity index 95% rename from install/kubernetes/backstage/README.md rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/README.md index e8d809c228..fc3d20733b 100644 --- a/install/kubernetes/backstage/README.md +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/README.md @@ -23,7 +23,7 @@ | app.resources | Kubernetes Pod resource requests/limits | `{}` | | app.nodeSelector | Node selectors for scheduling app/frontend pods | `{}` | | app.tolerations | Tolerations for scheduling app/frontend pods | `{}` | -| app.affinity | Affinity setttings for scheduling app/frontend pods | `{}` | +| app.affinity | Affinity settings for scheduling app/frontend pods | `{}` | ## Backend Values @@ -48,4 +48,4 @@ | backend.resources | Kubernetes Pod resource requests/limits | `{}` | | backend.nodeSelector | Node selectors for scheduling backend pods | `{}` | | backend.tolerations | Tolerations for scheduling backend pods | `{}` | -| backend.affinity | Affinity setttings for scheduling backend pods | `{}` | +| backend.affinity | Affinity settings for scheduling backend pods | `{}` | diff --git a/install/kubernetes/backstage/templates/_helpers.tpl b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/_helpers.tpl similarity index 100% rename from install/kubernetes/backstage/templates/_helpers.tpl rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/_helpers.tpl diff --git a/install/kubernetes/backstage/templates/deployment.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/deployment.yaml similarity index 100% rename from install/kubernetes/backstage/templates/deployment.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/deployment.yaml diff --git a/install/kubernetes/backstage/templates/ingress.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/ingress.yaml similarity index 100% rename from install/kubernetes/backstage/templates/ingress.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/ingress.yaml diff --git a/install/kubernetes/backstage/templates/service.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/service.yaml similarity index 100% rename from install/kubernetes/backstage/templates/service.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/service.yaml diff --git a/install/kubernetes/backstage/values.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/values.yaml similarity index 78% rename from install/kubernetes/backstage/values.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/values.yaml index fd4dd66b53..d91808ed28 100644 --- a/install/kubernetes/backstage/values.yaml +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/values.yaml @@ -1,12 +1,12 @@ app: enabled: true - nameOverride: "" - fullnameOverride: "" + nameOverride: '' + fullnameOverride: '' replicaCount: 1 serviceAccount: create: false - Name: "" - image: + Name: '' + image: repository: spotify/backstage tag: latest pullPolicy: Always @@ -15,20 +15,23 @@ app: port: 80 ingress: enabled: false - annotations: {} + annotations: + {} # kubernetes.io/ingress.class: "nginx" hosts: - - host: backstage.local - paths: - - / + - host: backstage.local + paths: + - / tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local imagePullSecrets: [] - podSecurityContext: {} + podSecurityContext: + {} # fsGroup: 2000 - securityContext: {} + securityContext: + {} # capabilities: # drop: # - ALL @@ -48,12 +51,12 @@ app: backend: enabled: false - nameOverride: "" - fullnameOverride: "" + nameOverride: '' + fullnameOverride: '' replicaCount: 1 serviceAccount: create: false - Name: "" + Name: '' image: repository: spotify/backstage-backend tag: latest @@ -63,20 +66,23 @@ backend: port: 7000 ingress: enabled: false - annotations: {} + annotations: + {} # kubernetes.io/ingress.class: "nginx" hosts: - - host: backstage.local - paths: - - /backend + - host: backstage.local + paths: + - /backend tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local imagePullSecrets: [] - podSecurityContext: {} + podSecurityContext: + {} # fsGroup: 2000 - securityContext: {} + securityContext: + {} # capabilities: # drop: # - ALL diff --git a/contrib/kubernetes/basic_kubernetes_example_with_helm/ingress.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/ingress.yaml new file mode 100644 index 0000000000..f93a814a0c --- /dev/null +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: backstage + labels: + app: backstage + component: ingress +spec: + rules: + - host: + http: + paths: + - backend: + serviceName: backstage + servicePort: frontend + path: / + - backend: + serviceName: backstage-backend + servicePort: backend + path: /backend diff --git a/install/kubernetes/service.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/service.yaml similarity index 71% rename from install/kubernetes/service.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/service.yaml index bbbc3d6e17..4d947b7afc 100644 --- a/install/kubernetes/service.yaml +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/service.yaml @@ -11,10 +11,10 @@ spec: app: backstage component: frontend ports: - - name: frontend - port: 80 - protocol: TCP - targetPort: app + - name: frontend + port: 80 + protocol: TCP + targetPort: app --- apiVersion: v1 kind: Service @@ -29,7 +29,7 @@ spec: app: backstage component: backend ports: - - name: backend - port: 7000 - protocol: TCP - targetPort: backend + - name: backend + port: 7000 + protocol: TCP + targetPort: backend diff --git a/docker-compose.yaml b/docker-compose.yaml index d32928b4e3..a9922cfb6c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,10 @@ # Make sure that before you # run the docker-compose that you have run -# $ yarn docker-build:all +# $ yarn docker-build version: '3' services: - frontend: - image: 'spotify/backstage:latest' - ports: - - '3000:80' - backend: + backstage: image: 'example-backend:latest' ports: - '7000:7000' diff --git a/docker/default.conf.template b/docker/default.conf.template index 29252f9625..23fdf271d3 100644 --- a/docker/default.conf.template +++ b/docker/default.conf.template @@ -11,6 +11,10 @@ server { try_files $uri /index.html; } + location /healthcheck { + return 204; + } + #error_page 404 /404.html; # redirect server error pages to the static page /50x.html diff --git a/docker/run.sh b/docker/run.sh index 469fdd08cf..fdb1742a07 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -16,13 +16,13 @@ function inject_config() { with_entries(select(.key | startswith("APP_CONFIG_")) | .key |= sub("APP_CONFIG_"; "")) | to_entries | reduce .[] as $item ( - {}; setpath($item.key | split("_"); $item.value | fromjson) + {}; setpath($item.key | split("_"); $item.value | try fromjson catch $item.value) )')" >&2 echo "Runtime app config: $config" local main_js - if ! main_js="$(grep -l __APP_INJECTED_RUNTIME_CONFIG__ /usr/share/nginx/html/*.chunk.js)"; then + if ! main_js="$(grep -l __APP_INJECTED_RUNTIME_CONFIG__ /usr/share/nginx/html/static/*.js)"; then echo "Runtime config already written" return fi diff --git a/docs/FAQ.md b/docs/FAQ.md index 7d568a840e..b4c4970ff8 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,6 +1,10 @@ -# FAQ +--- +id: FAQ +title: FAQ +description: All FAQ related to Backstage +--- -## Product FAQ: +## Product FAQ ### Can we call Backstage something different? So that it fits our company better? @@ -14,8 +18,7 @@ brand. No, but it can be! Backstage is designed to be a developer portal for all your infrastructure tooling, services, and documentation. So, it's not a monitoring platform — but that doesn't mean you can't integrate a monitoring tool into -Backstage by writing -[a plugin](/docs/FAQ.md#what-is-a-plugin-in-backstage). +Backstage by writing [a plugin](#what-is-a-plugin-in-backstage). ### How is Backstage licensed? @@ -36,12 +39,11 @@ more, read our blog post, Yes, we've already started releasing open source versions of some of the plugins we use here, and we'll continue to do so. -[Plugins](/docs/FAQ.md#what-is-a-plugin-in-backstage) are the -building blocks of functionality in Backstage. We have over 120 plugins inside -Spotify — many of those are specialized for our use, so will remain internal and -proprietary to us. But we estimate that about a third of our existing plugins -make good open source candidates. (And we'll probably end up writing some brand -new ones, too.) +[Plugins](#what-is-a-plugin-in-backstage) are the building blocks of +functionality in Backstage. We have over 120 plugins inside Spotify — many of +those are specialized for our use, so will remain internal and proprietary to +us. But we estimate that about a third of our existing plugins make good open +source candidates. (And we'll probably end up writing some brand new ones, too.) ### What's the roadmap for Backstage? @@ -66,7 +68,7 @@ valuable as you grow. Yes! The Backstage UI is built using Material-UI. With the theming capabilities of Material-UI, you are able to adapt the interface to your brand guidelines. -## Technical FAQ: +## Technical FAQ ### Why Material-UI? @@ -91,21 +93,31 @@ Node.js and GraphQL. ### What is the end-to-end user flow? The happy path story. There are three main user profiles for Backstage: the integrator, the -contributor, and the software engineer. +contributor, and the software engineer. -The **integrator** hosts the Backstage app and configures which plugins are available to use in the app. +The **integrator** hosts the Backstage app and configures which plugins are +available to use in the app. The **contributor** adds functionality to the app by writing plugins. -The **software engineer** uses the app's functionality and interacts with its plugins. +The **software engineer** uses the app's functionality and interacts with its +plugins. ### What is a "plugin" in Backstage? -Plugins are what provide the feature functionality in Backstage. They are used to integrate different systems into Backstage's frontend, so that the developer gets a consistent UX, no matter what tool or service is being accessed on the other side. +Plugins are what provide the feature functionality in Backstage. They are used +to integrate different systems into Backstage's frontend, so that the developer +gets a consistent UX, no matter what tool or service is being accessed on the +other side. -Each plugin is treated as a self-contained web app and can include almost any type of content. Plugins all use a common set of platform APIs and reusable UI components. Plugins can fetch data either from the backend or an API exposed through the proxy. +Each plugin is treated as a self-contained web app and can include almost any +type of content. Plugins all use a common set of platform APIs and reusable UI +components. Plugins can fetch data either from the backend or an API exposed +through the proxy. -Learn more about [the different components](https://github.com/spotify/backstage#overview) that make up Backstage. +Learn more about +[the different components](https://github.com/spotify/backstage#overview) that +make up Backstage. ### Do I have to write plugins in TypeScript? @@ -127,7 +139,15 @@ can browse and search for all available plugins. ### Which plugin is used the most at Spotify? -By far, our most-used plugin is our TechDocs plugin, which we use for creating technical documentation. Our philosophy at Spotify is to treat "docs like code", where you write documentation using the same workflow as you write your code. This makes it easier to create, find, and update documentation. We hope to release [the open source version](https://github.com/spotify/backstage/issues/687) in the future. (See also: "[Will Spotify's internal plugins be open sourced, too?](/docs/FAQ.md#will-spotifys-internal-plugins-be-open-sourced-too)" above) +By far, our most-used plugin is our TechDocs plugin, which we use for creating +technical documentation. Our philosophy at Spotify is to treat "docs like code", +where you write documentation using the same workflow as you write your code. +This makes it easier to create, find, and update documentation. We hope to +release +[the open source version](https://github.com/spotify/backstage/issues/687) in +the future. (See also: +"[Will Spotify's internal plugins be open sourced, too?](#will-spotifys-internal-plugins-be-open-sourced-too)" +above) ### Are you planning to have plugins baked into the repo? Or should they be developed in separate repos? @@ -135,10 +155,10 @@ Contributors can add open source plugins to the plugins directory in [this monorepo](https://github.com/spotify/backstage). Integrators can then configure which open source plugins are available to use in their instance of the app. Open source plugins are downloaded as npm packages published in the -open source repository. While we encourage using the open source model, we -know there are cases where contributors might want to experiment internally or -keep their plugins closed source. Contributors writing closed source plugins -should develop them in the plugins directory in their own Backstage repository. +open source repository. While we encourage using the open source model, we know +there are cases where contributors might want to experiment internally or keep +their plugins closed source. Contributors writing closed source plugins should +develop them in the plugins directory in their own Backstage repository. Integrators also configure closed source plugins locally from the monorepo. ### Any plans for integrating with other repository managers, such as GitLab or Bitbucket? @@ -149,7 +169,8 @@ stage. Hosting this project on GitHub does not exclude integrations with alternatives, such as [GitLab](https://github.com/spotify/backstage/issues?q=is%3Aissue+is%3Aopen+GitLab) or Bitbucket. We believe that in time there will be plugins that will provide -functionality for these tools as well. Hopefully, contributed by the community! Also note, implementations of Backstage can be hosted wherever you feel suits +functionality for these tools as well. Hopefully, contributed by the community! +Also note, implementations of Backstage can be hosted wherever you feel suits your needs best. ### Who maintains Backstage? @@ -165,8 +186,8 @@ maintains Backstage in your own environment. ### Does Spotify provide a managed version of Backstage? -No, this is not a service offering. We build the piece of software, and -someone in your infrastructure team is responsible for +No, this is not a service offering. We build the piece of software, and someone +in your infrastructure team is responsible for [deploying](https://github.com/spotify/backstage/blob/master/DEPLOYMENT.md) and maintaining it. @@ -186,16 +207,16 @@ Please report sensitive security issues via Spotify's No. Backstage does not collect any telemetry from any third party using the platform. Spotify, and the open source community, does have access to [GitHub Insights](https://github.com/features/insights), which contains -information such as contributors, commits, traffic, and dependencies. -Backstage is an open platform, but you are in control of your own data. You -control who has access to any data you provide to your version of Backstage and -who that data is shared with. +information such as contributors, commits, traffic, and dependencies. Backstage +is an open platform, but you are in control of your own data. You control who +has access to any data you provide to your version of Backstage and who that +data is shared with. ### Can Backstage be used to build something other than a developer portal? -Yes. The core frontend framework could be used for building any large-scale -web application where (1) multiple teams are building separate parts of the app, -and (2) you want the overall experience to be consistent. That being said, in +Yes. The core frontend framework could be used for building any large-scale web +application where (1) multiple teams are building separate parts of the app, and +(2) you want the overall experience to be consistent. That being said, in [Phase 2](https://github.com/spotify/backstage#project-roadmap) of the project we will add features that are needed for developer portals and systems for managing software ecosystems. Our ambition will be to keep Backstage modular. diff --git a/docs/README.md b/docs/README.md index 6763554afb..c63a12b589 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,98 +1,3 @@ -# Documentation structure +# Documentation -**Note!** This documentation structure is very much work in progress. If (when, -really 😆) you find broken links or missing content, please create an issue or, -better yet, a pull request. - -- Overview - - [What is Backstage?](overview/what-is-backstage.md) - - [Backstage architecture](overview/architecture-overview.md) - - [Architecture and terminology](overview/architecture-terminology.md) - - [Roadmap](overview/roadmap.md) - - Getting started - - [Running Backstage locally](getting-started/index.md) - - [Installation](getting-started/installation.md) - - [Local development](getting-started/development-environment.md) - - [Demo deployment](https://backstage-demo.roadie.io) - - Production deployments - - [Create an App](getting-started/create-an-app.md) - - App configuration - - [Configuring App with plugins](getting-started/configure-app-with-plugins.md) - - [Customize the look-and-feel of your App](getting-started/app-custom-theme.md) - - Deployment scenarios - - [Kubernetes](getting-started/deployment-k8s.md) - - [Other](getting-started/deployment-other.md) - - Features - - Software Catalog - - [Overview](features/software-catalog/index.md) - - [System model](features/software-catalog/system-model.md) - - [YAML File Format](features/software-catalog/descriptor-format.md) - - [Populating the catalog](features/software-catalog/populating.md) - - [Extending the model](features/software-catalog/extending-the-model.md) - - [External integrations](features/software-catalog/external-integrations.md) - - [API](features/software-catalog/api.md) - - Software creation templates - - [Overview](features/software-templates/index.md) - - [Adding templates](features/software-templates/adding-templates.md) - - Extending the Scaffolder: - - [Overview](features/software-templates/extending/index.md) - - [Create your own Templater](features/software-templates/extending/create-your-own-templater.md) - - [Create your own Publisher](features/software-templates/extending/create-your-own-publisher.md) - - [Create your own Preparer](features/software-templates/extending/create-your-own-preparer.md) - - Docs-like-code - - [Overview](features/techdocs/README.md) - - [Getting Started](features/techdocs/getting-started.md) - - [Concepts](features/techdocs/concepts.md) - - [Reading Documentation](features/techdocs/reading-documentation.md) - - [Writing Documentation](features/techdocs/writing-documentation.md) - - [Publishing Documentation](features/techdocs/publishing-documentation.md) - - [Contributing](features/techdocs/contributing.md) - - [Debugging](features/techdocs/debugging.md) - - [FAQ](features/techdocs/FAQ.md) - - Plugins - - [Overview](plugins/index.md) - - [Existing plugins](plugins/existing-plugins.md) - - [Creating a new plugin](plugins/create-a-plugin.md) - - [Developing a plugin](plugins/plugin-development.md) - - [Structure of a plugin](plugins/structure-of-a-plugin.md) - - Backends and APIs - - [Proxying](plugins/proxying.md) - - [Backstage backend plugin](plugins/backend-plugin.md) - - [Call existing API](plugins/call-existing-api.md) - - Testing - - [Overview](plugins/testing.md) - - Publishing - - [Open source and NPM](plugins/publishing.md) - - [Private/internal (non-open source)](plugins/publish-private.md) - - Authentication and identity - - [Overview](auth/index.md) - - [Add auth provider](auth/add-auth-provider.md) - - [Auth backend](auth/auth-backend.md) - - [OAuth](auth/oauth.md) - - [Glossary](auth/glossary.md) - - Designing for Backstage - - [Backstage Design Language System (DLS)](dls/design.md) - - [Storybook -- reusable UI components](http://storybook.backstage.io) - - [Contributing to Storybook](dls/contributing-to-storybook.md) - - [Figma resources](dls/figma.md) - - API references - - TypeScript API - - [Utilities](api/utility-apis.md) - - [createPlugin](reference/createPlugin.md) - - [createPlugin-feature-flags](reference/createPlugin-feature-flags.md) - - [createPlugin-router](reference/createPlugin-router.md) - - Backend APIs - - [Backend](api/backend.md) - - Tutorials - - [Overview](tutorials/index.md) - - Architecture Decision Records (ADRs) - - [Overview](architecture-decisions/index.md) - - [ADR001 - Architecture Decision Record (ADR) log](architecture-decisions/adr001-add-adr-log.md) - - [ADR002 - Default Software Catalog File Format](architecture-decisions/adr002-default-catalog-file-format.md) - - [ADR003 - Avoid Default Exports and Prefer Named Exports](architecture-decisions/adr003-avoid-default-exports.md) - - [ADR004 - Module Export Structure](architecture-decisions/adr004-module-export-structure.md) - - [ADR005 - Catalog Core Entities](architecture-decisions/adr005-catalog-core-entities.md) - - [ADR006 - Avoid React.FC and React.SFC](architecture-decisions/adr006-avoid-react-fc.md) - - [Contribute](../CONTRIBUTING.md) - - [Support](overview/support.md) - - [FAQ](FAQ.md) +The Backstage documentation is available at https://backstage.io/docs diff --git a/docs/api/backend.md b/docs/api/backend.md index e69de29bb2..0990343a01 100644 --- a/docs/api/backend.md +++ b/docs/api/backend.md @@ -0,0 +1,7 @@ +--- +id: backend +title: Backend +description: About Backend +--- + +## TODO diff --git a/docs/api/utility-apis.md b/docs/api/utility-apis.md index 2b15da0295..d192cbd6ee 100644 --- a/docs/api/utility-apis.md +++ b/docs/api/utility-apis.md @@ -1,4 +1,8 @@ -# Utility APIs +--- +id: utility-apis +title: Utility APIs +description: Backstage Utility APIs +--- ## Introduction @@ -14,12 +18,13 @@ Utility APIs. While the `createPlugin` API is focused on the initialization plugins and the app, the Utility APIs provide ways for plugins to communicate during their entire life cycle. -## Usage +## Consuming APIs Each Utility API is tied to an `ApiRef` instance, which is a global singleton object without any additional state or functionality, its only purpose is to reference Utility APIs. `ApiRef`s are create using `createApiRef`, which is -exported by `@backstage/core`. There are many predefined Utility APIs defined in +exported by `@backstage/core`. There are many +[predefined Utility APIs](../reference/utility-apis/README.md) defined in `@backstage/core`, and they're all exported with a name of the pattern `*ApiRef`, for example `errorApiRef`. @@ -51,47 +56,112 @@ from any component inside Backstage, including the ones in `@backstage/core`. The only requirement is that they are beneath the `AppProvider` in the react tree. -## Registering Utility API Implementations +## Supplying APIs -The Backstage App is responsible for providing implementations for all Utility -APIs required by plugins. The example app in this repo registers its APIs inside -[src/apis.ts](/packages/app/src/apis.ts). Here's an example of how to wire up -the `ErrorApi` inside an app: +### API Factories + +APIs are registered in the form of `ApiFactories`, which encapsulate the process +of instantiating an API. It is a collection of three things: the `ApiRef` of the +API to instantiate, a list of all required dependencies, and a factory function +that returns a new API instance. + +For example, this is the default `ApiFactory` for the `ErrorApi`: ```ts -import { - ApiRegistry, - createApp, - alertApiRef, - errorApiRef, - AlertApiForwarder, - ErrorApiForwarder, - ErrorAlerter, -} from '@backstage/core'; - -const builder = ApiRegistry.builder(); - -// The alert API is a self-contained implementation that shows alerts to the user. -const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); - -// The error API uses the alert API to send error notifications to the user. -builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); - -const app = createApp({ - apis: apiBuilder.build(), - // ... other config +createApiFactory({ + api: errorApiRef, + deps: { alertApi: alertApiRef }, + factory: ({ alertApi }) => + new ErrorAlerter(alertApi, new ErrorApiForwarder()), }); ``` -The `ApiRegistry` is used to register all Utility APIs in the app and associate -them with `ApiRef`s. It implements the `ApiHolder` interface, which enables it -to provide an API implementation given an `ApiRef`. +In this example the `errorApiRef` is our API, which encapsulates the `ErrorApi` +type. The `alertApiRef` is our single dependency, which we give the name +`alertApi`, and is then passed on to the factory function, which returns an +implementation of the `ErrorApi`. -Note that our `ErrorApi` implementation depends on another Utility API, the -`AlertApi`. This is the method with which APIs can depend on other APIs, using -manual dependency injection at the initialization of the app. In general, if you -want to depend on another Utility API in an implementation of an API, you import -the type for that API and make it a constructor parameter. +The `createApiFactory` function is a thin wrapper that enables TypeScript type +inference. You may notice that there are no type annotations in the above +example, and that is because we're able to infer all types from the `ApiRef`s. +TypeScript will make sure that the return value of the `factory` function +matches the type embedded in `api`'s `ApiRef`, in this case the `ErrorApi`. It +will also match the types between the `deps` and the parameters of the `factory` +function, again using the type embedded within the `ApiRef`s. + +## Registering API Factories + +The responsibility for adding Utility APIs to a Backstage app lies in three +different locations: the Backstage core library, each plugin included in the +app, and the app itself. + +### Core APIs + +Starting with the Backstage core library, it provides implementation for all of +the core APIs. The core APIs are the ones exported by `@backstage/core`, such as +the `errorApiRef` and `configApiRef`. You can find a full list of them +[here](../reference/utility-apis/README.md). + +The core APIs are loaded for any app created with `createApp` from +`@backstage/core`, which means that there is no step that needs to be taken to +include these APIs in an app. + +### Plugin APIs + +In addition to the core APIs, plugins can define and export their own APIs. +While doing so they should usually also provide default implementations of their +own APIs, for example, the `catalog` plugin exports `catalogApiRef`, and also +supplies a default `ApiFactory` of that API using the `CatalogClient`. There is +one restriction to plugin-provided API Factories: plugins may not supply +factories for core APIs, trying to do so will cause the app to crash. + +Plugins supply their APIs through the `apis` option of `createPlugin`, for +example: + +```ts +export const plugin = createPlugin({ + id: 'techdocs', + apis: [ + createApiFactory({ + api: techdocsStorageApiRef, + deps: { configApi: configApiRef }, + factory({ configApi }) { + return new TechDocsStorageApi({ + apiOrigin: configApi.getString('techdocs.storageUrl'), + }); + }, + }), + ], +}); +``` + +### App APIs + +Lastly, the app itself is the final point where APIs can be added, and what has +the final say in what APIs will be loaded at runtime. The app may override the +factories for any of the core or plugin APIs, with the exception of the config, +app theme, and identity APIs. These are static APIs that are tied into the +`createApp` implementation, and therefore not possible to override. + +Overriding APIs is useful for apps that want to switch out behavior to tailor it +to their environment. In some cases plugins may also export multiple +implementations of the same API, where they each have their own different +requirements on for example backend storage and surrounding environment. + +Supplying APIs to the app works just like for plugins: + +```ts +const app = createApp({ + apis: [ + /* ApiFactories */ + ], + // ... other options +}); +``` + +A common pattern is to export a list of all APIs from `apis.ts`, next to +`App.tsx`. See the [example app in this repo](../../packages/app/src/apis.ts) +for an example. ## Custom implementations of Utility APIs @@ -119,19 +189,33 @@ implement the `ErrorApi`, as it is checked by the type embedded in the ## Defining custom Utility APIs -The pattern for plugins defining their own Utility APIs is not fully established -yet. The current way is for the plugin to export its own `ApiRef` and type for -the API, along with one or more implementations. It is then up to the app to -import, and register those APIs. See for example the -[lighthouse](/plugins/lighthouse/src/api.ts) or -[graphiql](/plugins/graphiql/src/lib/api/types.ts) plugins for examples of this. +Plugins are free to define their own Utility APIs. Simply define the TypeScript +interface for the API, and create an `ApiRef` using `createApiRef` exported from +`@backstage/core`. Also be sure to provide at least one implementation of the +API, and to declare a default factory for the API in `createPlugin`. -The goal is to make this process a bit smoother, but that requires work in other -parts of Backstage, like configuration management. So it remains as a TODO. If -you have more questions regarding this, or have an idea for an API that you want -to share outside your plugin, hit us up in -[GitHub issues](https://github.com/spotify/backstage/issues/new/choose) or the -[Backstage Discord server](https://discord.gg/EBHEGzX). +Custom Utility APIs can be either public or private, which it is up to the +plugin to choose. Private APIs do not expose an external API surface, and it's +therefore possible to make breaking changes to the API without affecting other +users of the plugin. If an API is made public however, it opens up for other +plugins to make use of the API, and it also makes it possible for users for your +plugin to override the API in the app. It is however important to maintain +backwards compatibility of public APIs, as you may otherwise break apps that are +using your plugin. + +To make an API public, simply export the `ApiRef` of the API, and any associated +types. To make an API private, just avoid exporting the `ApiRef`, but still be +sure to supply a default factory to `createPlugin`. + +Private APIs are useful for plugins that want to depend on other APIs outside of +React components, but not have to expose an entire API surface to maintain. When +using private APIs, it is fine to use the `typeof` of an implementing class as +the type parameter passed to `createApiRef`, while public APIs should always +define a separate TypeScript interface type. + +Plugins may depend on APIs from other plugins, both in React components and as +dependencies to API factories. Do however be sure to not cause circular +dependencies between plugins. ## Architecture @@ -152,7 +236,7 @@ The figure below shows the relationship between fooApiRef.
-Figure showing the relationship between utility APIs, the apps that provide them, and the plugins that consume them +Figure showing the relationship between utility APIs, the apps that provide them, and the plugins that consume them
The current method for connecting Utility API providers and consumers is via the diff --git a/docs/architecture-decisions/adr001-add-adr-log.md b/docs/architecture-decisions/adr001-add-adr-log.md index e6ab39aea4..16783367ab 100644 --- a/docs/architecture-decisions/adr001-add-adr-log.md +++ b/docs/architecture-decisions/adr001-add-adr-log.md @@ -1,4 +1,8 @@ -# ADR001: Architecture Decision Record (ADR) log +--- +id: adrs-adr001 +title: ADR001: Architecture Decision Record (ADR) log +description: Architecture Decision Record (ADR) logs as a reference point for the team +--- | Created | Status | | ---------- | ------ | diff --git a/docs/architecture-decisions/adr002-default-catalog-file-format.md b/docs/architecture-decisions/adr002-default-catalog-file-format.md index 00c8b0947d..f4fd3a1158 100644 --- a/docs/architecture-decisions/adr002-default-catalog-file-format.md +++ b/docs/architecture-decisions/adr002-default-catalog-file-format.md @@ -1,4 +1,8 @@ -# ADR002: Default Software Catalog File Format +--- +id: adrs-adr002 +title: ADR002: Default Software Catalog File Format +description: Architecture Decision Record (ADR) log on Default Software Catalog File Format +--- | Created | Status | | ---------- | ------ | @@ -64,7 +68,7 @@ metadata: lifecycle: production example.com/service-discovery-name: frobsawesome annotations: - circleci.com/project-slug: gh/example-org/frobs-awesome + circleci.com/project-slug: github/example-org/frobs-awesome spec: type: service ``` diff --git a/docs/architecture-decisions/adr003-avoid-default-exports.md b/docs/architecture-decisions/adr003-avoid-default-exports.md index 7becebaa4b..8634a19632 100644 --- a/docs/architecture-decisions/adr003-avoid-default-exports.md +++ b/docs/architecture-decisions/adr003-avoid-default-exports.md @@ -1,4 +1,8 @@ -# ADR003: Avoid Default Exports and Prefer Named Exports +--- +id: adrs-adr003 +title: ADR003: Avoid Default Exports and Prefer Named Exports +description: Architecture Decision Record (ADR) log on Avoid Default Exports and Prefer Named Exports +--- | Created | Status | | ---------- | ------ | diff --git a/docs/architecture-decisions/adr004-module-export-structure.md b/docs/architecture-decisions/adr004-module-export-structure.md index 54132773c9..12408abac1 100644 --- a/docs/architecture-decisions/adr004-module-export-structure.md +++ b/docs/architecture-decisions/adr004-module-export-structure.md @@ -1,4 +1,8 @@ -# ADR004: Module Export Structure +--- +id: adrs-adr004 +title: ADR004: Module Export Structure +description: Architecture Decision Record (ADR) log on Module Export Structure +--- | Created | Status | | ---------- | ------ | diff --git a/docs/architecture-decisions/adr005-catalog-core-entities.md b/docs/architecture-decisions/adr005-catalog-core-entities.md index d607dc62fb..f91698c5ff 100644 --- a/docs/architecture-decisions/adr005-catalog-core-entities.md +++ b/docs/architecture-decisions/adr005-catalog-core-entities.md @@ -1,4 +1,8 @@ -# ADR005: Catalog Core Entities +--- +id: adrs-adr005 +title: ADR005: Catalog Core Entities +description: Architecture Decision Record (ADR) log on Catalog Core Entities +--- | Created | Status | | ---------- | ------ | @@ -18,7 +22,7 @@ Backstage should eventually support the following core entities: - **Resources** are physical or virtual infrastructure needed to operate a component -![Catalog Core Entities](catalog-core-entities.png) +![Catalog Core Entities](../assets/architecture-decisions/catalog-core-entities.png) For now, we'll start by only implementing support for the Component entity in the Backstage catalog. This can later be extended to APIs, Resources and other diff --git a/docs/architecture-decisions/adr006-avoid-react-fc.md b/docs/architecture-decisions/adr006-avoid-react-fc.md index 21e6d5b637..bea36712d7 100644 --- a/docs/architecture-decisions/adr006-avoid-react-fc.md +++ b/docs/architecture-decisions/adr006-avoid-react-fc.md @@ -1,4 +1,8 @@ -# ADR006: Avoid React.FC and React.SFC +--- +id: adrs-adr006 +title: ADR006: Avoid React.FC and React.SFC +description: Architecture Decision Record (ADR) log on Avoid React.FC and React.SFC +--- ## Context diff --git a/docs/architecture-decisions/adr007-use-msw-to-mock-service-requests.md b/docs/architecture-decisions/adr007-use-msw-to-mock-service-requests.md new file mode 100644 index 0000000000..6b18f67d40 --- /dev/null +++ b/docs/architecture-decisions/adr007-use-msw-to-mock-service-requests.md @@ -0,0 +1,64 @@ +--- +id: adrs-adr007 +title: ADR007: Use MSW to mock http requests +description: Architecture Decision Record (ADR) log on Use MSW to mock http requests +--- + +## Context + +Network request mocking can be a total pain sometimes, in all different types of +tests, unit tests to e2e tests always have their own implementation of mocking +these requests. There's been traction in the outer community towards using this +library to mock network requests by using an express style declaration for +routes. react-testing-library suggests using this library instead of mocking +fetch directly wether this be in a browser or in node. + +https://github.com/mswjs/msw + +## Decision + +Moving forward, we have decided that any `fetch` or `XMLHTTPRequest` that +happens, should be mocked by using `msw`. + +Here is an example: + +```ts +import { setupWorker, rest } from 'msw'; + +const worker = setupWorker( + rest.get('*/user/:userId', (req, res, ctx) => { + return res( + ctx.json({ + firstName: 'John', + lastName: 'Maverick', + }), + ); + }), +); + +// Start the Mock Service Worker +worker.start(); +``` + +and in a more real life scenario, taken from +[CatalogClient.test.ts](https://github.com/spotify/backstage/blob/f3245c4f8f0b6b2625c4a6d5d50161b612fb4757/plugins/catalog/src/api/CatalogClient.test.ts) + +```ts +beforeEach(() => { + server.use( + rest.get(`${mockApiOrigin}${mockBasePath}/entities`, (_, res, ctx) => { + return res(ctx.json(defaultResponse)); + }), + ); +}); + +it('should entities from correct endpoint', async () => { + const entities = await client.getEntities(); + expect(entities).toEqual(defaultResponse); +}); +``` + +## Consequences + +- A little more code to write +- Gradually will replace the codebase with `msw` diff --git a/docs/architecture-decisions/adr008-default-catalog-file-name.md b/docs/architecture-decisions/adr008-default-catalog-file-name.md new file mode 100644 index 0000000000..c57d10c0b1 --- /dev/null +++ b/docs/architecture-decisions/adr008-default-catalog-file-name.md @@ -0,0 +1,26 @@ +--- +id: adrs-adr008 +title: ADR008: Default Catalog File Name +description: Architecture Decision Record (ADR) log on Default Catalog File Name +--- + +## Background + +While the spec for the catalog file format is well described in +[ADR002](./adr002-default-catalog-file-format.md), guidance was note provided as +to the name of the catalog file. + +Following discussion in +[Issue 1822](https://github.com/spotify/backstage/pull/1822#pullrequestreview-461253670), +a decision was made. + +## Name + +The catalog file should be named + +```shell +catalog-info.yaml +``` + +This name is a default, **not a requirement**. The catalog file will work with +Backstage irregardless of its name. diff --git a/docs/architecture-decisions/index.md b/docs/architecture-decisions/index.md index 7762a9c827..ddf2805e75 100644 --- a/docs/architecture-decisions/index.md +++ b/docs/architecture-decisions/index.md @@ -1,6 +1,11 @@ -# Architecture Decision Records (ADR) +--- +id: adrs-overview +title: Architecture Decision Records (ADR) +sidebar_label: Overview +description: Overview of Architecture Decision Records (ADR) +--- -The substantial architecture decisions made in the Backstage project lives here. +The substantial architecture decisions made in the Backstage project live here. For more information about ADRs, when to write them, and why, please see [this blog post](https://engineering.atspotify.com/2020/04/14/when-should-i-write-an-architecture-decision-record/). @@ -19,7 +24,10 @@ Records should be stored under the `architecture-decisions` directory. - Submit a pull request - Address and integrate feedback from the community - Eventually, assign a number -- Add the full path of the ADR to the [`mkdocs.yml`](/mkdocs.yml) +- Add the path of the ADR to the microsite sidebar in + [`sidebars.json`](https://github.com/spotify/backstage/blob/master/microsite/sidebars.json) +- Add the path of the ADR to the + [`mkdocs.yml`](https://github.com/spotify/backstage/blob/master/mkdocs.yml) - Merge the pull request ## Superseding an ADR diff --git a/docs/architecture-decisions/catalog-core-entities.png b/docs/assets/architecture-decisions/catalog-core-entities.png similarity index 100% rename from docs/architecture-decisions/catalog-core-entities.png rename to docs/assets/architecture-decisions/catalog-core-entities.png diff --git a/docs/overview/architecture-overview/backstage-typical-architecture.png b/docs/assets/architecture-overview/backstage-typical-architecture.png similarity index 100% rename from docs/overview/architecture-overview/backstage-typical-architecture.png rename to docs/assets/architecture-overview/backstage-typical-architecture.png diff --git a/docs/overview/architecture-overview/circle-ci-plugin-architecture.png b/docs/assets/architecture-overview/circle-ci-plugin-architecture.png similarity index 100% rename from docs/overview/architecture-overview/circle-ci-plugin-architecture.png rename to docs/assets/architecture-overview/circle-ci-plugin-architecture.png diff --git a/docs/overview/architecture-overview/circle-ci.png b/docs/assets/architecture-overview/circle-ci.png similarity index 100% rename from docs/overview/architecture-overview/circle-ci.png rename to docs/assets/architecture-overview/circle-ci.png diff --git a/docs/overview/architecture-overview/containerised.png b/docs/assets/architecture-overview/containerised.png similarity index 100% rename from docs/overview/architecture-overview/containerised.png rename to docs/assets/architecture-overview/containerised.png diff --git a/docs/overview/architecture-overview/core-vs-plugin-components-highlighted.png b/docs/assets/architecture-overview/core-vs-plugin-components-highlighted.png similarity index 100% rename from docs/overview/architecture-overview/core-vs-plugin-components-highlighted.png rename to docs/assets/architecture-overview/core-vs-plugin-components-highlighted.png diff --git a/docs/overview/architecture-overview/lighthouse-plugin-architecture.png b/docs/assets/architecture-overview/lighthouse-plugin-architecture.png similarity index 100% rename from docs/overview/architecture-overview/lighthouse-plugin-architecture.png rename to docs/assets/architecture-overview/lighthouse-plugin-architecture.png diff --git a/docs/overview/architecture-overview/lighthouse-plugin.png b/docs/assets/architecture-overview/lighthouse-plugin.png similarity index 100% rename from docs/overview/architecture-overview/lighthouse-plugin.png rename to docs/assets/architecture-overview/lighthouse-plugin.png diff --git a/docs/overview/architecture-overview/tech-radar-plugin-architecture.png b/docs/assets/architecture-overview/tech-radar-plugin-architecture.png similarity index 100% rename from docs/overview/architecture-overview/tech-radar-plugin-architecture.png rename to docs/assets/architecture-overview/tech-radar-plugin-architecture.png diff --git a/docs/overview/architecture-overview/tech-radar-plugin.png b/docs/assets/architecture-overview/tech-radar-plugin.png similarity index 100% rename from docs/overview/architecture-overview/tech-radar-plugin.png rename to docs/assets/architecture-overview/tech-radar-plugin.png diff --git a/docs/auth/oauth-popup-flow.svg b/docs/assets/auth/oauth-popup-flow.svg similarity index 98% rename from docs/auth/oauth-popup-flow.svg rename to docs/assets/auth/oauth-popup-flow.svg index 4d9e79787a..2132903783 100644 --- a/docs/auth/oauth-popup-flow.svg +++ b/docs/assets/auth/oauth-popup-flow.svg @@ -1,5 +1,5 @@ -OAuth Consent and Refresh FlowBrowserBrowserPopup WindowPopup Windowauth-backend pluginauth-backend pluginConsent ScreenConsent ScreenOAuth ProviderOAuth ProviderComponents on page ask for anaccess token with greaterscope than the existing session.Open popupGET /auth/<provider>/start?scope=some%20scopesRedirect to consent screen withrandom nonce in OAuth state andshort-lived cookie with the same nonce.GET /consent_url?redirect_uri=<redirect_uri>?nonce=<n>where redirect_uri=<app-origin>/auth/<provider>/handler/frameUser consents toaccess the new scope.Redirect to given redirect URL, with authorization codeGET /auth/<provider>/handler/frame?code=<c>&nonce=<n>Request includes the previously set none cookieVerify that the nonce in the cookiematches the nonce in the OAuth stateAuthorization CodeClient IDClient SecretVerify and generate tokensAccess Token(ID Token)(Refresh Token)ScopeExpire TimeSmall HTML page with inlined response payloadStore Refresh Token in HTTP-only cookiepostMessage() with tokens and info or errorClose selfA later point when a refreshis needed. Either because ofa reload or an expiring session.GET /auth/<provider>/tokenRefresh Token cookie includedRefresh TokenClient IDClient SecretAccess Token(ID Token)ScopeExpire TimeTokens and info - -![](oauth-popup-flow.svg) diff --git a/docs/conf/defining.md b/docs/conf/defining.md new file mode 100644 index 0000000000..bea03e4e44 --- /dev/null +++ b/docs/conf/defining.md @@ -0,0 +1,19 @@ +--- +id: defining +title: Defining Configuration for your Plugin +description: Documentation on Defining Configuration for your Plugin +--- + +There is currently no tooling support or helpers for defining plugin +configuration. But it's on the roadmap. + +Meanwhile, document the config values that you are reading in your plugin +README. + +## Format + +When defining configuration for your plugin, keep keys camelCased and stick to +existing casing conventions such as `baseUrl`. + +It is also usually best to prefer objects over arrays, as it makes it possible +to override individual values using separate files or environment variables. diff --git a/docs/conf/index.md b/docs/conf/index.md new file mode 100644 index 0000000000..83eff50a7d --- /dev/null +++ b/docs/conf/index.md @@ -0,0 +1,52 @@ +--- +id: index +title: Static Configuration in Backstage +description: Documentation on Static Configuration in Backstage +--- + +## Summary + +Backstage ships with a flexible configuration system that provides a simple way +to configure Backstage apps and plugins for both local development and +production deployments. It helps get you up and running fast while adapting +Backstage for your specific environment. It also serves as a tool for plugin +authors to use to make it simple to pick up and install a plugin, while still +allowing for customization. + +## Supplying Configuration + +Configuration is stored in `app-config.yaml` files, with support for suffixes +such as `app-config.production.yaml` to override values for specific +environments. The configuration files themselves contain plain YAML, but with +support for loading in secrets from various sources using a `$secret` key. + +It is also possible to supply configuration through environment variables, for +example `APP_CONFIG_app_baseUrl=https://staging.example.com`. However these +should be used sparingly, usually just for temporary overrides during +development or small tweaks to be able to reuse deployment artifacts in +different environments. + +The configuration is shared between the frontend and backend, meaning that +values that are common between the two only needs to be defined once. Such as +the `backend.baseUrl`. + +For more details, see [Writing Configuration](./writing.md). + +## Reading Configuration + +As a plugin developer, you likely end up wanting to define configuration that +you want users of your plugin to supply, as well as reading that configuration +in frontend and backend plugins. For more details, see +[Reading Configuration](./reading.md) and +[Defining Configuration](./defining.md). + +## Further Reading + +More details are provided in dedicated sections of the documentation. + +- [Reading Configuration](./reading.md): How to read configuration in your + plugin. +- [Writing Configuration](./writing.md): How to provide configuration for your + Backstage deployment. +- [Defining Configuration](./defining.md): How to define configuration for users + of your plugin. diff --git a/docs/conf/reading.md b/docs/conf/reading.md new file mode 100644 index 0000000000..a43ec8c026 --- /dev/null +++ b/docs/conf/reading.md @@ -0,0 +1,132 @@ +--- +id: reading +title: Reading Backstage Configuration +description: Documentation on Reading Backstage Configuration +--- + +## Config API + +There's a common configuration API for by both frontend and backend plugins. An +API reference can be found [here](../reference/utility-apis/Config.md). + +The configuration API is tailored towards failing fast in case of missing or bad +config. That's because configuration errors can always be considered programming +mistakes, and will fail deterministically. + +### Type Safety + +The methods for reading primitive values are typed, and validate that type at +runtime. For example `getNumber()` requires the underlying value to be a number, +and there will be no attempt to coerce other types into the desired one. If +`getNumber()` receives a string value, it will throw an error, explaining where +the bad config came from, and what the desired and actual types where. + +### Reading Nested Configuration + +The backing configuration data is a nested JSON structure, meaning there will be +object, within objects, arrays within objects, and so on. There are a couple of +different ways to access nested values when reading configuration, but the +primary one is to use dot-separated paths. + +For example, given the following configuration: + +```yaml +app: + baseUrl: http://localhost:3000 +``` + +We can access the `baseUrl` using `config.getString('app.baseUrl')`. Because of +this syntax, configuration keys are not allowed to contain dots. In fact, +configuration keys are validated using the following RegEx: +`/^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i`. + +Another option of accessing the `baseUrl` value is to create a sub-view of the +configuration, `config.getConfig('app').getString('baseUrl')`. When reading out +single values the dot-path pattern is preferred, but creating sub-views can be +useful for when you want to pass on parts of configuration to be read out by a +separate function. For example, given something like + +```yaml +my-plugin: + items: + a: + title: Item A + path: /a + b: + title: Item B + path: /b +``` + +You can get the list of all items using the `.keys()` method, and then pass on +each sub-view to be handled individually. + +```ts +for (const itemKey of config.keys('my-plugin.items')) { + const itemConfig = config.getConfig(`my-plugin.items`).getConfig(key); + const item = createItemFromConfig(itemConfig); +} +``` + +Another option for iterating through configuration keys is to call +`config.get('my-plugin.items')`, which simply returns the JSON structure for +that position without any validation. This can be handy to use sometimes, +especially if you're passing on config to an external library. There's a clear +benefit to the sub-view approach though, which is that the user will receive +much more detailed and relevant error messages. For example, if +`itemConfig.getString('title')` fails in the above example because a boolean was +supplied, the user will receive an error message with the full path, e.g. +`my-plugin.items.b.title`, as well as the name of the config file with the bad +value. + +Note that no matter what method is used for reading out nested config, the same +merging rules apply. You will always get the same value for any way of accessing +nested config: + +```ts +// Equivalent as long as a.b.c exists and is a string +config.getString('a.b.c'); +config.getConfig('a.b').getString('c'); +config.get('a').b.c; +``` + +### Required vs Optional Configuration + +Reading configuration can be divided into two categories: required, and +optional. When reading optional configuration you use the optional methods such +as `getOptionalString`. These methods will simply return `undefined` if +configuration values are missing, allowing the called to fall back to default +values. The optional methods still validate types however, so receiving a string +in a call to `config.getOptionalNumber` will still throw an error. + +A good pattern for reading optional configuration values is to use the `??` +operator. For example: + +```ts +const title = config.getOptionalString('my-plugin.title') ?? 'My Plugin'; +``` + +To read required configuration, simply use the methods without `Optional`, for +example `getString`. These will throw an error if there is no value available. + +## Accessing ConfigApi in Frontend Plugins + +The [ConfigApi](../reference/utility-apis/Config.md) in the frontend is a +[UtilityApi](../api/utility-apis.md). It's accessible as usual via the +`configApiRef` exported from `@backstage/core`. + +Depending on the config api in another API is slightly different though, as the +`ConfigApi` implementation is supplied via the App itself and not instantiated +like other APIs. See +[packages/app/src/apis.ts](https://github.com/spotify/backstage/blob/244eef851f5aa19f91c7c9b5c12d5df95cf482ca/packages/app/src/apis.ts#L66) +for an example of how this wiring is done. + +For standalone plugin setups in `dev/index.ts`, register a factory with a +statically mocked implementation of the config API. Use the `ConfigReader` from +`@backstage/config` to create an instance and register it for the `configApiRef` +from `@backstage/core`. + +## Accessing ConfigApi in Backend Plugins + +In backend plugins the configuration is passed in via options from the main +backend package. See for example +[packages/backend/src/plugins/auth.ts](https://github.com/spotify/backstage/blob/244eef851f5aa19f91c7c9b5c12d5df95cf482ca/packages/backend/src/plugins/auth.ts#L23). diff --git a/docs/conf/writing.md b/docs/conf/writing.md new file mode 100644 index 0000000000..6d338e586d --- /dev/null +++ b/docs/conf/writing.md @@ -0,0 +1,161 @@ +--- +id: writing +title: Writing Backstage Configuration Files +description: Documentation on Writing Backstage Configuration Files +--- + +## File Format + +Configuration is stored in YAML format in `app-config.yaml` files, looking +something like this: + +```yaml +app: + title: Backstage Example App + baseUrl: http://localhost:3000 + +backend: + listen: 0.0.0.0:7000 + baseUrl: http://localhost:7000 + +organization: + name: Spotify + +proxy: + /my/api: + target: https://example.com/api/ + changeOrigin: true + pathRewrite: + ^/proxy/my/api/: / +``` + +Configuration files are typically checked in and stored in the repo that houses +the rest of the Backstage application. + +## Environment Variable Overrides + +Individual configuration values can be overridden using environment variables +prefixed with `APP_CONFIG_`. Everything following that prefix in the environment +variable name will be used as the config key, with `_` replaced by `.`. For +example, to override the `app.baseUrl` value, set the `APP_CONFIG_app_baseUrl` +environment variable to the desired value. + +The value of the environment variable is parsed as JSON, but it will fall back +to being interpreted as a string if it fails to parse. Note that if you for +example want to pass on the string `"false"`, you need to wrap it in double +quotes, e.g. `export APP_CONFIG_example='"false"'`. + +While it may be tempting to use environment variable overrides for supplying a +lot of configuration values, we recommend using them sparingly. Try to stick to +using configuration files, and only use environment variables for things like +reusing deployment artifacts across staging and production environments. + +Note that environment variables work for frontend configuration too. They are +picked up by the serve tasks of `@backstage/cli` for local development, and are +injected by the entrypoint of the nginx container serving the frontend in a +production build. + +## File Resolution + +It is possible to have multiple configuration files, both to support different +environments, but also to define configuration that is local to specific +packages. + +All `app-config.yaml` files inside the monorepo root and package root are +considered, as are files with additional `local` and environment affixes such as +`development`, for example `app-config.local.yaml`, +`app-config.production.yaml`, and `app-config.development.local.yaml`. Which +environment config files are loaded is determined by the `NODE_ENV` environment +variable. Local configuration files are always loaded, but are meant for local +development overrides and should typically be `.gitignore`'d. + +All loaded configuration files are merged together using the following rules: + +- Configurations have different priority, higher priority means you replace + values from configurations with lower priority. +- Primitive values are completely replaced, as are arrays and all of their + contents. +- Objects are merged together deeply, meaning that if any of the included + configs contain a value for a given path, it will be found. + +The priority of the configurations is determined by the following rules, in +order: + +- Configuration from the `APP_CONFIG_` environment variables has the highest + priority, followed by files. +- Files inside package directories have higher priority than those in the root + directory. +- Files with environment affixes have higher priority than ones without. +- Files with the `local` affix have higher priority than ones without. + +## Secrets + +Secrets are supported via a special `$secret` key, which in turn provides a +number of different ways to read in secrets. To load a configuration value as a +secret, supply an object with a single `$secret` key, and within that supply an +object that describes how the secret is loaded. For example, the following will +read the config key `backend.mySecretKey` from the environment variable +`MY_SECRET_KEY`: + +```yaml +backend: + mySecretKey: + $secret: + env: MY_SECRET_KEY +``` + +With the above configuration, calling `config.getString('backend.mySecretKey')` +will return the value of the environment variable `MY_SECRET_KEY` when the +backend started up. All secrets are loaded at startup, so changing the contents +of secret files or environment variables will not be reflected at runtime. + +Note that secrets will never be included in the frontend bundle or development +builds. When loading configuration you have to explicitly enable reading of +secrets, which is only done for the backend configuration. + +As hinted at, secrets can be loaded from a bunch of different sources, and can +be extended with more. Below is a list of the currently supported methods for +loading secrets. + +### Env Secrets + +This reads a secret from an environment variable. For example, the following +config loads the secret from the `MY_SECRET` env var. + +```yaml +$secret: + env: MY_SECRET +``` + +### File Secrets + +This reads a secret from the entire contents of a file. The file path is +relative to the `app-config.yaml` the defines the secrets. For example, the +following reads the contents of `my-secret.txt` relative to the config file +itself: + +```yaml +$secret: + file: ./my-secret.txt +``` + +### Data File Secrets + +This reads secrets from a path within a JSON-like data file. The file path +behaves similar to file secrets, but in addition a `path` is used to point to a +specific value inside the file. Supported file extensions are `.json`, `.yaml`, +and `.yml`. For example, the following would read out `my-secret-key` from +`my-secrets.json`: + +```yaml +$secret: + data: ./my-secrets.json + path: deployment.key + +# my-secrets.json +{ + "deployment": { + "key": "my-secret-key" + } +} +``` diff --git a/docs/dls/contributing-to-storybook.md b/docs/dls/contributing-to-storybook.md index 7dbe72f6d3..058f4a23a7 100644 --- a/docs/dls/contributing-to-storybook.md +++ b/docs/dls/contributing-to-storybook.md @@ -1,4 +1,11 @@ -# Contributing to Storybook +--- +id: contributing-to-storybook +title: Contributing to Storybook +description: Documentation on How to Contribute to Storybook +--- + +You find our storybook at +[http://backstage.io/storybook](http://backstage.io/storybook) ## Creating a new Story @@ -26,7 +33,7 @@ core Go to `packages/storybook`, run `yarn install` and install the dependencies, then run the following on your command line: `yarn start` -![](running-storybook.png) +![](../assets/dls/running-storybook.png) _You should see a log like the image above._ @@ -34,4 +41,4 @@ If everything worked out, your server will be running on **port 6006**, go to your browser and navigate to `http://localhost:6006/`. You should be able to navigate and see the Storybook page. -![](storybook-page.png) +![](../assets/dls/storybook-page.png) diff --git a/docs/dls/design.md b/docs/dls/design.md index e3816bd2e5..fa3a0c159e 100644 --- a/docs/dls/design.md +++ b/docs/dls/design.md @@ -1,4 +1,10 @@ -![header](designheader.png) +--- +id: design +title: Design +description: Documentation on Design +--- + +![header](../assets/dls/designheader-updated.png) Much like Backstage Open Source, this is a _living_ document! We'll keep this updated as we evolve our practices! @@ -60,7 +66,7 @@ that is shaped by user experience and user interface decisions made by our Backstage Design Team. Also note, we encourage you to take the core experience we’ve crafted and add custom theming to better represent your organization! -![dls](DLS.png) +![dls](../assets/dls/DLS.png) ## ✅ Our Priorities @@ -106,18 +112,17 @@ picked up by our team as something to be added to our design system. ## ✏️ Resources -**[Storybook](http://storybook.backstage.io/)** - where you can view our +**[Storybook](http://backstage.io/storybook)** - where you can view our components. If you’d like to help build up our design system, you can also add components we’ve designed to the Storybook as well. +**[Figma](https://www.figma.com/@backstage)** - we're stoked to be using Figma +Community to share our design assets. You can duplicate our UI Kit and design +your own plugin for Backstage. + **[Discord](https://discord.gg/EBHEGzX)** - all design questions should be directed to the _#design_ channel. -**Documentation** - -- Patterns (stay tuned) -- Figma files/libraries (stay tuned) - ## 🔮 Future ### Contributions from designers diff --git a/docs/dls/figma.md b/docs/dls/figma.md index 8fe6ea8d0f..c5a33d15c2 100644 --- a/docs/dls/figma.md +++ b/docs/dls/figma.md @@ -1 +1,9 @@ -We have a [Figma component library](https://www.figma.com/@backstage) that you can use to build your own plugins for Backstage. +--- +id: figma +title: Figma +description: Documentation on using Figma to build your own plugins for +Backstage +--- + +We have a [Figma component library](https://www.figma.com/@backstage) that you +can use to build your own plugins for Backstage. diff --git a/docs/features/software-catalog/api.md b/docs/features/software-catalog/api.md index e69de29bb2..f7c7b19112 100644 --- a/docs/features/software-catalog/api.md +++ b/docs/features/software-catalog/api.md @@ -0,0 +1,7 @@ +--- +id: software-catalog-api +title: API +description: Documentation on Software Catalog API +--- + +## TODO diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..c98bd73ca3 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,120 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +description: Documentation on Software Catalog Configuration +--- + +## Processors + +The catalog makes use of so called processors to perform all kinds of ingestion +tasks, such as reading raw entity data from a remote source, parsing it, +transforming it, and validating it. These processors are configured under the +`catalog.processors` key. + +### Processor: github + +The `github` processor is responsible for fetching entity data from files on +GitHub or GitHub Enterprise. The configuration for this processor lives under +`catalog.processors.github`. Example: + +```yaml +catalog: + processors: + github: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + - target: https://ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_PRIVATE_TOKEN +``` + +The main subkey is `providers`, where you can list the various GitHub compatible +providers you want to be able to fetch data from. Each entry is a structure with +up to four elements: + +- `target` (required): The string prefix of the location target that you want to + match on, with no trailing slash. For GitHub, it should be exactly + `https://github.com`. +- `token` (optional): An authentication token as expected by GitHub. If + supplied, it will be passed along with all calls to this provider, both API + and raw. If it is not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is github, you can leave it out + to be inferred automatically. For a GitHub Enterprise installation, it is + commonly at `https://api.` or `https:///api/v3`. +- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is public GitHub, you can leave + it out to be inferred automatically. For a GitHub Enterprise installation, it + is commonly at `https://api.` or `https:///api/v3`. + +You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for +public GitHub, for which we can infer them). The `apiBaseUrl` will always be +preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be +preferred. + +If you do not supply a public GitHub provider, one will be added automatically, +silently at startup for convenience. So you only have to list it if you want to +supply a token for it - and if you do, you can also leave out the `apiBaseUrl` +and `rawBaseUrl` fields. + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component`, `API` and `Location`. In order to allow entities of other kinds to +be added, you need to add rules to the catalog. Rules are added either in a +separate `catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API, Location, Template] + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + rules: + - allow: [Group] +``` + +We are able to add entities of kind `Component`, `API`, `Location`, or +`Template` from any location, and `Group` entities from the `org-data.yaml`, +which will also be read as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for the default kinds if you want +those to still be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 68cee2352c..301d80630c 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -1,4 +1,10 @@ -# Descriptor Format of Catalog Entities +--- +id: descriptor-format +title: Descriptor Format of Catalog Entities +sidebar_label: YAML File Format +description: Documentation on Descriptor Format of Catalog Entities which +describes the default data shape and semantics of catalog entities +--- This section describes the default data shape and semantics of catalog entities. @@ -14,6 +20,8 @@ humans. However, the structure and semantics is the same in both cases. - [Common to All Kinds: The Envelope](#common-to-all-kinds-the-envelope) - [Common to All Kinds: The Metadata](#common-to-all-kinds-the-metadata) - [Kind: Component](#kind-component) +- [Kind: Template](#kind-template) +- [Kind: API](#kind-api) ## Overall Shape Of An Entity @@ -28,7 +36,7 @@ software catalog API. "annotations": { "backstage.io/managed-by-location": "file:/tmp/component-info.yaml", "example.com/service-discovery": "artistweb", - "circleci.com/project-slug": "gh/example-org/artist-website" + "circleci.com/project-slug": "github/example-org/artist-website" }, "description": "The place to be, for great artists", "etag": "ZjU2MWRkZWUtMmMxZS00YTZiLWFmMWMtOTE1NGNiZDdlYzNk", @@ -36,6 +44,7 @@ software catalog API. "labels": { "system": "public-websites" }, + "tags": ["java"], "name": "artist-web", "uid": "2152f463-549d-4d8d-a94d-ce2b7676c6e2" }, @@ -59,7 +68,9 @@ metadata: system: public-websites annotations: example.com/service-discovery: artistweb - circleci.com/project-slug: gh/example-org/artist-website + circleci.com/project-slug: github/example-org/artist-website + tags: + - java spec: type: website lifecycle: production @@ -80,7 +91,7 @@ The root envelope object has the following structure. ### `apiVersion` and `kind` [required] The `kind` is the high level entity type being described. -[ADR005](/docs/architecture-decisions/adr005-catalog-core-entities.md) describes +[ADR005](../../architecture-decisions/adr005-catalog-core-entities.md) describes a number of core kinds that plugins can know of and understand, but an organization using Backstage is free to also add entities of other kinds to the catalog. @@ -226,6 +237,23 @@ The `backstage.io/` prefix is reserved for use by Backstage core components. Values can be of any length, but are limited to being strings. +There is a list of [well-known annotations](well-known-annotations.md), but +anybody is free to add more annotations as they see fit. + +### `tags` [optional] + +A list of single-valued strings, for example to classify catalog entities in +various ways. This is different to the labels in metadata, as labels are +key-value pairs. + +The values are user defined, for example the programming language used for the +component, like `java` or `go`. + +This field is optional, and currently has no special semantics. + +Each tag must be sequences of `[a-zA-Z0-9]` separated by `-`, at most 63 +characters in total. + ## Kind: Component Describes the following entity kind: @@ -252,6 +280,8 @@ spec: type: website lifecycle: production owner: artist-relations@example.com + implementsApis: + - artist-api ``` In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) @@ -265,7 +295,7 @@ Exactly equal to `backstage.io/v1alpha1` and `Component`, respectively. The type of component as a string, e.g. `website`. This field is required. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -279,9 +309,9 @@ The current set of well-known and common values for this field is: ### `spec.lifecycle` [required] -The lifecyle state of the component, e.g. `production`. This field is required. +The lifecycle state of the component, e.g. `production`. This field is required. -The software catalog accepts any lifecycle value, but an organisation should +The software catalog accepts any lifecycle value, but an organization should take great care to establish a proper taxonomy for these. The current set of well-known and common values for this field is: @@ -312,14 +342,22 @@ Apart from being a string, the software catalog leaves the format of this field open to implementers to choose. Most commonly, it is set to the ID or email of a group of people in an organizational structure. +### `spec.implementsApis` [optional] + +Links APIs that are implemented by the component, e.g. `artist-api`. This field +is optional. + +The software catalog expects a list of one or more strings that references the +names of other entities of the `kind` `API`. + ## Kind: Template Describes the following entity kind: -| Field | Value | -| -------------------- | ----------------------- | -| `apiVersion` | `backstage.io/v1alpha1` | -| `Kind: Templatekind` | `Template` | +| Field | Value | +| ------------ | ----------------------- | +| `apiVersion` | `backstage.io/v1alpha1` | +| `kind` | `Template` | A Template describes a skeleton for use with the Scaffolder. It is used for describing what templating library is supported, and also for documenting the @@ -337,8 +375,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: cookiecutter @@ -385,7 +423,7 @@ potentially search and group templates by these tags. The type of component as a string, e.g. `website`. This field is optional but recommended. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -422,3 +460,114 @@ specify relative to the `template.yaml` definition. This is also particularly useful when you have multiple template definitions in the same repository but only a single `template.yaml` registered in backstage. + +## Kind: API + +Describes the following entity kind: + +| Field | Value | +| ------------ | ----------------------- | +| `apiVersion` | `backstage.io/v1alpha1` | +| `kind` | `API` | + +An API describes an interface that can be exposed by a component. The API can be +defined in different formats, like [OpenAPI](https://swagger.io/specification/), +[AsyncAPI](https://www.asyncapi.com/docs/specifications/latest/), +[GraphQL](https://graphql.org/learn/schema/), +[gRPC](https://developers.google.com/protocol-buffers), or other formats. + +Descriptor files for this kind may look as follows. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: artist-api + description: Retrieve artist details +spec: + type: openapi + lifecycle: production + owner: artist-relations@example.com + definition: | + openapi: "3.0.0" + info: + version: 1.0.0 + title: Artist API + license: + name: MIT + servers: + - url: http://artist.spotify.net/v1 + paths: + /artists: + get: + summary: List all artists + ... +``` + +In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) +shape, this kind has the following structure. + +### `apiVersion` and `kind` [required] + +Exactly equal to `backstage.io/v1alpha1` and `API`, respectively. + +### `spec.type` [required] + +The type of the API definition as a string, e.g. `openapi`. This field is +required. + +The software catalog accepts any type value, but an organization should take +great care to establish a proper taxonomy for these. Tools including Backstage +itself may read this field and behave differently depending on its value. For +example, an OpenAPI type API may be displayed using an OpenAPI viewer tooling in +the Backstage interface. + +The current set of well-known and common values for this field is: + +- `openapi` - An API definition in YAML or JSON format based on the + [OpenAPI](https://swagger.io/specification/) version 2 or version 3 spec. +- `asyncapi` - An API definition based on the + [AsyncAPI](https://www.asyncapi.com/docs/specifications/latest/) spec. +- `grpc` - An API definition based on + [Protocol Buffers](https://developers.google.com/protocol-buffers) to use with + [gRPC](https://grpc.io/). + +### `spec.lifecycle` [required] + +The lifecycle state of the API, e.g. `production`. This field is required. + +The software catalog accepts any lifecycle value, but an organization should +take great care to establish a proper taxonomy for these. + +The current set of well-known and common values for this field is: + +- `experimental` - an experiment or early, non-production API, signaling that + users may not prefer to consume it over other more established APIs, or that + there are low or no reliability guarantees +- `production` - an established, owned, maintained API +- `deprecated` - an API that is at the end of its lifecycle, and may disappear + at a later point in time + +### `spec.owner` [required] + +The owner of the API, e.g. `artist-relations@example.com`. This field is +required. + +In Backstage, the owner of an API is the singular entity (commonly a team) that +bears ultimate responsibility for the API, and has the authority and capability +to develop and maintain it. They will be the point of contact if something goes +wrong, or if features are to be requested. The main purpose of this field is for +display purposes in Backstage, so that people looking at catalog items can get +an understanding of to whom this API belongs. It is not to be used by automated +processes to for example assign authorization in runtime systems. There may be +others that also develop or otherwise touch the API, but there will always be +one ultimate owner. + +Apart from being a string, the software catalog leaves the format of this field +open to implementers to choose. Most commonly, it is set to the ID or email of a +group of people in an organizational structure. + +### `spec.definition` [required] + +The definition of the API, based on the format defined by `spec.type`. This +field is required. diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index e69de29bb2..4e8a357ea7 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -0,0 +1,43 @@ +--- +id: extending-the-model +title: Extending the model +description: Documentation on Extending the model +--- + +Backstage natively supports tracking of the following component +[`type`](descriptor-format.md)'s: + +- Services +- Websites +- Libraries +- Documentation +- Other + +![](../../assets/software-catalog/bsc-extend.png) + +Since these types are likely not the only kind of software you will want to +track in Backstage, it is possible to + +It is possible to add your own software types that fits your organization's data +model. Inside Spotify our model has grown significantly over the years, and now +includes ML models, Apps, data pipelines and many more. + +## Adding a new type + +TODO: Describe what changes are needed to add a new type that shows up in the +catalog. + +## The Other type + +It might be tempting to put software that doesn't fit into any of the existing +types into Other. There are a few reasons why we advice against this; firstly, +we have found that it is preferred to match the conceptual model that your +engineers have when describing your software. Secondly, Backstage helps your +engineers manage their software by integrating the infrastructure tooling +through plugins. Different plugins are used for managing different types of +components. + +For example, the +[Lighthouse plugin](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) +only makes sense for Websites. The more specific you can be in how you model +your software, the easier it is to provide plugins that are contextual. diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index e69de29bb2..00a9d95fc2 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -0,0 +1,13 @@ +--- +id: external-integrations +title: External integrations +description: Documentation on External integrations to integrate systems +with Backstage +--- + +Backstage natively supports storing software components in +[metadata YAML files](descriptor-format.md). However, companies that already +have an existing system for keeping track of software and its owners can +integrate such systems with Backstage. + +TODO: Describe the API contract. diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index 4925f32e1a..49a490d78b 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -1,4 +1,10 @@ -# Backstage Service Catalog (alpha) +--- +id: software-catalog-overview +title: Backstage Service Catalog (alpha) +sidebar_label: Overview +description: The Backstage Service Catalog — actually, a software catalog, since +it includes more than just services +--- ## What is a Service Catalog? @@ -6,21 +12,132 @@ The Backstage Service Catalog — actually, a software catalog, since it include more than just services — is a centralized system that keeps track of ownership and metadata for all the software in your ecosystem (services, websites, libraries, data pipelines, etc). The catalog is built around the concept of -[metadata yaml files](../../architecture-decisions/adr002-default-catalog-file-format.md#format) -stored together with the code, which are then harvested and visualized in -Backstage. +[metadata YAML files](descriptor-format.md) stored together with the code, which +are then harvested and visualized in Backstage. ![service-catalog](https://backstage.io/blog/assets/6/header.png) -We have also found that the service catalog is a great way to organise the -infrastructure tools you use to manage the software as well. This is how -Backstage creates one developer portal for all your tools. Rather than asking -teams to jump between different infrastructure UI’s (and incurring additional -cognitive overhead each time they make a context switch), most of these tools -can be organised around the entities in the catalog. +## How it works -## Using the Service Catalog +Backstage and the Backstage Service Catalog makes it easy for one team to manage +10 services — and makes it possible for your company to manage thousands of +them. -TODO +More specifically, the Service Catalog enables two main use-cases: -![](service-catalog-home.png) +1. Helping teams manage and maintain the software they own. Teams get a uniform + view of all their software; services, libraries, websites, ML models — you + name it, Backstage knows all about it. +2. Makes all the software in your company, and who owns it, discoverable. No + more orphan software hiding in the dark corners of your software ecosystem. + +## Getting Started + +The Software Catalog is available to browse on the start page at `/`. If you've +followed [Installing in your Backstage App](./installation.md) in your separate +App or [Getting Started with Backstage](../../getting-started) for this repo, +you should be able to browse the catalog at `http://localhost:3000`. + +![](../../assets/software-catalog/service-catalog-home.png) + +## Adding components to the catalog + +The source of truth for the components in your service catalog are +[metadata YAML files](descriptor-format.md) stored in source control (GitHub, +GitHub Enterprise, GitLab, ...). + +There are 3 ways to add components to the catalog: + +1. Manually register components +2. Creating new components through Backstage +3. Integrating with an [external source](external-integrations.md) + +### Manually register components + +Users can register new components by going to `/create` and clicking the +**REGISTER EXISTING COMPONENT** button: + +![](../../assets/software-catalog/bsc-register-1.png) + +Backstage expects the full URL to the YAML in your source control. Example: + +```bash +https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +_More examples can be found +[here](https://github.com/spotify/backstage/tree/master/packages/catalog-model/examples)._ + +![](../../assets/software-catalog/bsc-register-2.png) + +It is important to note that any kind of software can be registered in +Backstage. Even if the software is not maintained by your company (SaaS +offering, for example) it is still useful to create components for tracking +ownership. + +### Creating new components through Backstage + +All software created through the +[Backstage Software Templates](../software-templates/index.md) are automatically +registered in the catalog. + +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + +### Updating component metadata + +Teams owning the components are responsible for maintaining the metadata about +them, and do so using their normal Git workflow. + +![](../../assets/software-catalog/bsc-edit.png) + +Once the change has been merged, Backstage will automatically show the updated +metadata in the service catalog after a short while. + +## Finding software in the catalog + +By default the service catalog shows components owned by the team of the logged +in user. But you can also switch to _All_ to see all the components across your +company's software ecosystem. Basic inline _search_ and _column filtering_ makes +it easy to browse a big set of components. + +![](../../assets/software-catalog/bsc-search.png) + +## Starring components + +For easy and quick access to components you visit frequently, Backstage supports +_starring_ of components: + +![](../../assets/software-catalog/bsc-starred.png) + +## Integrated tooling through plugins + +The service catalog is a great way to organize the infrastructure tools you use +to manage the software. This is how Backstage creates one developer portal for +all your tools. Rather than asking teams to jump between different +infrastructure UIs (and incurring additional cognitive overhead each time they +make a context switch), most of these tools can be organized around the entities +in the catalog. + +![tools](https://backstage.io/blog/assets/20-05-20/tabs.png) + +The Backstage platform can be customized by incorporating +[existing open source plugins](https://github.com/spotify/backstage/tree/master/plugins), +or by [building your own](../../plugins/index.md). + +## Links + +- [[Blog post] Backstage Service Catalog released in alpha](https://backstage.io/blog/2020/06/22/backstage-service-catalog-alpha) diff --git a/docs/features/software-catalog/installation.md b/docs/features/software-catalog/installation.md new file mode 100644 index 0000000000..a825020eb6 --- /dev/null +++ b/docs/features/software-catalog/installation.md @@ -0,0 +1,197 @@ +--- +id: installation +title: Installing in your Backstage App +description: Documentation on How to install Backstage Plugin +--- + +The catalog plugin comes in two packages, `@backstage/plugin-catalog` and +`@backstage/plugin-catalog-backend`. Each has their own installation steps, +outlined below. + +## Installing @backstage/plugin-catalog + +> **Note that if you used `npx @backstage/create-app`, the plugin may already be +> present** + +The catalog frontend plugin should be installed in your `app` package, which is +created as a part of `@backstage/create-app`. To install the package, run: + +```bash +cd packages/app +yarn add @backstage/plugin-catalog +``` + +Make sure the version of `@backstage/plugin-catalog` matches the version of +other `@backstage` packages. You can update it in `packages/app/package.json` if +it doesn't. + +### Adding the Plugin to your `packages/app` + +Add the following entry to the head of your `packages/app/src/plugins.ts`: + +```ts +export { plugin as CatalogPlugin } from '@backstage/plugin-catalog'; +``` + +Add the following to your `packages/app/src/apis.ts`: + +```ts +import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; + +// Inside the ApiRegistry builder function ... + +builder.add( + catalogApiRef, + new CatalogClient({ + apiOrigin: backendUrl, + basePath: '/catalog', + }), +); +``` + +Where `backendUrl` is the `backend.baseUrl` from config, i.e. +`const backendUrl = config.getString('backend.baseUrl')`. + +The catalog components depend on a number of other +[Utility APIs](../../api/utility-apis.md) to function, including at least the +`ErrorApi` and `StorageApi`. You can find an example of how to install these in +your app +[here](https://github.com/spotify/backstage/blob/61c3a7e5b750dc7c059ef16b188594d31b2c04c2/packages/app/src/apis.ts#L80). + +## Gotchas that we will fix + +Since the catalog plugin currently ships with a sentry plugin `InfoCard` +installed by default, you'll need to set `sentry.organization` in your +`app-yaml.yaml`. For example: + +```yaml +sentry: + organization: Acme Corporation +``` + +If you've created an app with an older version of `@backstage/create-app` or +`@backstage/cli create-app`, be sure to remove the Welcome plugin from the app, +as that will conflict with the catalog routes. + +## Installing @backstage/plugin-catalog-backend + +> **Note that if you used `npx @backstage/create-app`, the plugin may already be +> present** + +The catalog backend should be installed in your `backend` package, which is +created as a part of `@backstage/create-app`. To install the package, run: + +```bash +cd packages/backend +yarn add @backstage/plugin-catalog-backend +``` + +Make sure the version of `@backstage/plugin-catalog-backend` matches the version +of other `@backstage` packages. You can update it in +`packages/backend/package.json` if it doesn't. + +### Adding the Plugin to your `packages/backend` + +You'll need to add the plugin to the `backend`'s router. You can do this by +creating a file called `packages/backend/src/plugins/catalog.ts` with the +following contents to get you up and running quickly. + +```ts +import { + createRouter, + DatabaseEntitiesCatalog, + DatabaseLocationsCatalog, + DatabaseManager, + HigherOrderOperations, + LocationReaders, + runPeriodically, +} from '@backstage/plugin-catalog-backend'; +import { PluginEnvironment } from '../types'; +import { useHotCleanup } from '@backstage/backend-common'; + +export default async function createPlugin({ + logger, + database, +}: PluginEnvironment) { + const locationReader = new LocationReaders(logger); + + const db = await DatabaseManager.createDatabase(database, { logger }); + const entitiesCatalog = new DatabaseEntitiesCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + const higherOrderOperation = new HigherOrderOperations( + entitiesCatalog, + locationsCatalog, + locationReader, + logger, + ); + + useHotCleanup( + module, + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000), + ); + + return await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + logger, + }); +} +``` + +Once the `catalog.ts` router setup file is in place, add the router to +`packages/backend/src/index.ts`: + +```ts +import catalog from './plugins/catalog'; + +const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + +const service = createServiceBuilder(module) + .loadConfig(configReader) + /** several different routers */ + .addRouter('/catalog', await catalog(catalogEnv)); +``` + +### Adding Entries to the Catalog + +At this point the catalog backend is installed in your backend package, but you +will not have any entities loaded. + +To get up and running and try out some templates quickly, you can add some of +our example templates through static configuration. Add the following to the +`catalog.locations` section in your `app-config.yaml`: + +```yaml +catalog: + locations: + # Backstage Example Component + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml +``` + +### Running the Backend + +Finally, start up the backend with the new configuration: + +```bash +cd packages/backend +yarn start +``` + +If you've also set up the frontend plugin, so you should be ready to go browse +the catalog at [localhost:3000](http://localhost:3000) now! diff --git a/docs/features/software-catalog/system-model.md b/docs/features/software-catalog/system-model.md index e69de29bb2..44dac13d65 100644 --- a/docs/features/software-catalog/system-model.md +++ b/docs/features/software-catalog/system-model.md @@ -0,0 +1,117 @@ +--- +id: system-model +title: System Model +description: Documentation on System Model +--- + +We believe that a strong shared understanding and terminology around software +and resources leads to a better Backstage experience. + +_This description originates from +[this RFC](https://github.com/spotify/backstage/issues/390). Note that some of +the concepts are not yet supported in Backstage._ + +## Core Entities + +We model software in the Backstage catalogue using these three core entities +(further explained below): + +- **Components** are individual pieces of software + +- **APIs** are the boundaries between different components + +- **Resources** are physical or virtual infrastructure needed to operate a + component + +![](../../assets/software-catalog/software-model-core-entities.png) + +### Component + +A component is a piece of software, for example a mobile feature, web site, +backend service or data pipeline (list not exhaustive). A component can be +tracked in source control, or use some existing open source or commercial +software. + +A component can implement APIs for other components to consume. In turn it might +depend on APIs implemented by other components, or resources that are attached +to it at runtime. + +### API + +APIs form an important (maybe the most important) abstraction that allows large +software ecosystems to scale. Thus, APIs are a first class citizen in the +Backstage model and the primary way to discover existing functionality in the +ecosystem. + +APIs are implemented by components and form boundaries between components. They +might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a data schema (eg +Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by +components need to be in a known machine-readable format so we can build further +tooling and analysis on top. + +APIs have a visibility: they are either public (making them available for any +other component to consume), restricted (only available to a whitelisted set of +consumers), or private (only available within their system). As public APIs are +going to be the primary way interaction between components, Backstage supports +documenting, indexing and searching all APIs so we can browse them as +developers. + +### Resource + +Resources are the infrastructure a component needs to operate at runtime, like +BigTable databases, Pub/Sub topics, S3 buckets or CDNs. Modelling them together +with components and systems will better allow us to visualize resource +footprint, and create tooling around them. + +## Ecosystem Modeling + +A large catalogue of components, APIs and resources can be highly granular and +hard to understand as a whole. It might thus be convenient to further categorize +these entities using the following (optional) concepts: + +- **Systems** are a collection of entities that cooperate to perform some + function +- **Domains** relate entities and systems to part of the business + +### System + +With increasing complexity in software, systems form an important abstraction +level to help us reason about software ecosystems. Systems are a useful concept +in that they allow us to ignore the implementation details of a certain +functionality for consumers, while allowing the owning team to make changes as +they see fit (leading to low coupling). + +A system, in this sense, is a collection of resources and components that +exposes one or several public APIs. The main benefit of modelling a system is +that it hides its resources and private APIs between the components for any +consumers. This means that as the owner, you can evolve the implementation, in +terms of components and resources, without your consumers being able to notice. +Typically, a system will consist of at most a handful of components (see Domain +for a grouping of systems). + +For example, a playlist management system might encapsulate a backend service to +update playlists, a backend service to query them, and a database to store them. +It could expose an RPC API, a daily snapshots dataset, and an event stream of +playlist updates. + +### Domain + +While systems are the basic level of encapsulation for related entities, it is +often useful to group a collection of systems that share terminology, domain +models, metrics, KPIs, business purpose, or documentation, i.e. they form a +bounded context. + +For example, it would make sense if the different systems in the “Payments” +domain would come with some documentation on how to accept payments for a new +product or use-case, share the same entity types in their APIs, and integrate +well with each other. Other domains could be “Content Ingestion”, “Ads” or +“Search”. + +## Current status + +Backstage currently supports Components and APIs. + +## Links + +- [Original RFC](https://github.com/spotify/backstage/issues/390) +- [YAML file format](../../architecture-decisions/adr002-default-catalog-file-format.md) diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md new file mode 100644 index 0000000000..ba5b491e07 --- /dev/null +++ b/docs/features/software-catalog/well-known-annotations.md @@ -0,0 +1,137 @@ +--- +id: well-known-annotations +title: Well-known Annotations on Catalog Entities +sidebar_label: Well-known Annotations +description: Documentation on lists a number of well known Annotations, that +have defined semantics. They can be attached to catalog entities and consumed +by plugins as needed +--- + +This section lists a number of well known +[annotations](descriptor-format.md#annotations-optional), that have defined +semantics. They can be attached to catalog entities and consumed by plugins as +needed. + +## Annotations + +This is a (non-exhaustive) list of annotations that are known to be in active +use. + +### backstage.io/managed-by-location + +```yaml +# Example: +metadata: + annotations: + backstage.io/managed-by-location: github:http://github.com/spotify/backstage/catalog-info.yaml +``` + +The value of this annotation is a so called location reference string, that +points to the source from which the entity was originally fetched. This +annotation is added automatically by the catalog as it fetches the data from a +registered location, and is not meant to normally be written by humans. The +annotation may point to any type of generic location that the catalog supports, +so it cannot be relied on to always be specifically of type `github`, nor that +it even represents a single file. Note also that a single location can be the +source of many entities, so it represents a many-to-one relationship. + +The format of the value is `:`. Note that the target may also +contain colons, so it is not advisable to naively split the value on `:` and +expecting a two-item array out of it. The format of the target part is +type-dependent and could conceivably even be an empty string, but the separator +colon is always present. + +### backstage.io/techdocs-ref + +```yaml +# Example: +metadata: + annotations: + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git +``` + +The value of this annotation is a location reference string (see above). If this +annotation is specified, it is expected to point to a repository that the +TechDocs system can read and generate docs from. + +### jenkins.io/github-folder + +```yaml +# Example: +metadata: + annotations: + jenkins.io/github-folder: folder-name/job-name +``` + +The value of this annotation is the path to a job on Jenkins, that builds this +entity. + +Specifying this annotation may enable Jenkins related features in Backstage for +that entity. + +### github.com/project-slug + +```yaml +# Example: +metadata: + annotations: + github.com/project-slug: spotify/backstage +``` + +The value of this annotation is the so-called slug that identifies a project on +[GitHub](https://github.com) that is related to this entity. It is on the format +`/`, and is the same as can be seen in the URL location +bar of the browser when viewing that project. + +Specifying this annotation will enable GitHub related features in Backstage for +that entity. + +### sentry.io/project-slug + +```yaml +# Example: +metadata: + annotations: + sentry.io/project-slug: pump-station +``` + +The value of this annotation is the so-called slug (or alternatively, the ID) of +a [Sentry](https://sentry.io) project within your organization. The organization +slug is currently not configurable on a per-entity basis, but is assumed to be +the same for all entities in the catalog. + +Specifying this annotation may enable Sentry related features in Backstage for +that entity. + +### rollbar.com/project-slug + +```yaml +# Example: +metadata: + annotations: + rollbar.com/project-slug: spotify/pump-station +``` + +The value of this annotation is the so-called slug (or alternatively, the ID) of +a [Rollbar](https://rollbar.com) project within your organization. The value can +be the format of `[organization]/[project-slug]` or just `[project-slug]`. When +the organization slug is omitted the `app-config.yaml` will be used as a +fallback (`rollbar.organization` followed by `organization.name`). + +Specifying this annotation may enable Rollbar related features in Backstage for +that entity. + +## Deprecated Annotations + +The following annotations are deprecated, and only listed here to aid in +migrating away from them. + +### backstage.io/github-actions-id + +This annotation was used for a while to enable the GitHub Actions feature. This +is now instead using the [github.com/project-slug](#github-com-project-slug) +annotation, with the same value format. + +## Links + +- [Descriptor Format: annotations](descriptor-format.md#annotations-optional) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index e929ec4a97..fad5c68e41 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -1,4 +1,8 @@ -# Adding your own Templates +--- +id: adding-templates +title: Adding your own Templates +description: Documentation on Adding your own Templates +--- Templates are stored in the **Service Catalog** under a kind `Template`. The minimum that the a template skeleton needs is a `template.yaml` but it would be @@ -10,24 +14,25 @@ A simple `template.yaml` definition might look something like this: apiVersion: backstage.io/v1alpha1 kind: Template metadata: - # unique name per namespace for the template + # unique name per namespace for the template name: react-ssr-template - # title of the template + # title of the template title: React SSR Template - # a description of the template - description: Next.js application skeleton for creating isomorphic web applications. + # a description of the template + description: + Next.js application skeleton for creating isomorphic web applications. # some tags to display in the frontend - tags: - - Recommended - - React + tags: + - recommended + - react spec: # which templater key to use in the templaters builder templater: cookiecutter - # what does this template create + # what does this template create type: website - # if the template is not in the current directory where this definition is kept then specfiy + # if the template is not in the current directory where this definition is kept then specfiy path: './template' - # the schema for the form which is displayed in the frontend. + # the schema for the form which is displayed in the frontend. # should follow JSON schema for forms: https://jsonforms.io/ schema: required: @@ -39,7 +44,7 @@ spec: type: string description: Unique name of the component description: - title: Description + title: Description type: string description: Description of the component ``` @@ -50,11 +55,34 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. -Currently the catalog supports loading definitions from Github + Local Files. To +Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. -For loading from a file the following command should work when the backend is +You can add the template files to the catalog through +[static location configuration](../software-catalog/configuration.md#static-location-configuration), +for example + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + rules: + - allow: [Template] +``` + +Templates can also be added by posting the to the catalog directly. Note that if +you're doing this, you need to configure the catalog to allow template entities +to be ingested from any source, for example: + +```yaml +catalog: + rules: + - allow: [Component, API, Template] +``` + +For loading from a file, the following command should work when the backend is running: ```sh @@ -65,7 +93,7 @@ curl \ --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" ``` -If loading from a git location, you can run the following +If loading from a Git location, you can run the following ```sh curl \ @@ -78,13 +106,6 @@ curl \ This should then have added the catalog, and also should now be listed under the create page at http://localhost:3000/create. -Alternatively, if you want to get setup with some mock templates that are -already provided for you, you can run the following to load those templates: - -``` -yarn lerna run mock-data -``` - The `type` field which is chosen in the request to add the `template.yaml` to the Service Catalog here, will be come the `PreparerKey` which will be used to select the `Preparer` when creating a job. diff --git a/docs/features/software-templates/extending/create-your-own-preparer.md b/docs/features/software-templates/extending/create-your-own-preparer.md index 4e45bef535..14c75b0ffb 100644 --- a/docs/features/software-templates/extending/create-your-own-preparer.md +++ b/docs/features/software-templates/extending/create-your-own-preparer.md @@ -1,4 +1,8 @@ -# Create your own Preparer +--- +id: extending-preparer +title: Create your own Preparer +description: Documentation on Creating your own Preparer +--- Preparers are responsible for reading the location of the definition of a [Template Entity](../../software-catalog/descriptor-format.md#kind-template) and @@ -51,7 +55,7 @@ The `protocol` is set on the when added to the service catalog. You can see more about this `PreparerKey` here in [Register your own template](../adding-templates.md) -**note:** Currently the catalog supports loading definitions from Github + Local +**note:** Currently the catalog supports loading definitions from GitHub + Local Files, which translate into the two `PreparerKeys` `file` and `github`. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 8baf604e01..25d7bcf676 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -1,21 +1,25 @@ -# Create your own Publisher +--- +id: extending-publisher +title: Create your own Publisher +description: Documentation on Creating your own Publisher +--- Publishers are responsible for pushing and storing the templated skeleton after the values have been templated by the `Templater`. See [Create your own templater](./create-your-own-templater.md) for more info. -They recieve a directory or location where the templater has sucessfully run on, -and is now ready to store somewhere. They also get given some other options -which are sent from the frontend, such as the `storePath` which is a string of -where the frontend thinks we should save this templated folder. +They receive a directory or location where the templater has sucessfully run and +is now ready to store somewhere. They also are given some other options which +are sent from the frontend, such as the `storePath` which is a string of where +the frontend thinks we should save this templated folder. Currently we provide the following `publishers`: - `github` This publisher is passed through to the `createRouter` function of the -`@spotify/plugin-scaffolder-backend`. Currently only one publisher is supported, -but PR's are always welcome. +`@spotify/plugin-scaffolder-backend`. Currently, only one publisher is +supported, but PR's are always welcome. An full example backend can be found [here](https://github.com/spotify/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index 947190c5e5..36c362b9c3 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -1,18 +1,22 @@ -# Creating your own Templater +--- +id: extending-templater +title: Creating your own Templater +description: Documentation on Creating your own Templater +--- Templaters are responsible for taking the directory path for the skeleton returned by the preparers, and then executing the templating command on top of the file and returning the completed template path. This may or may not be the same directory as the input directory. -They also recieve additional values from the frontend, which can be used to +They also receive additional values from the frontend, which can be used to interpolate into the skeleton files. Currently we provide the following templaters: - `cookiecutter` -This templater is added the `TemplaterBuilder` and then passed into the +This templater is added to the `TemplaterBuilder` and then passed into the `createRouter` function of the `@spotify/plugin-scaffolder-backend` An full example backend can be found @@ -45,7 +49,7 @@ This `TemplaterKey` is used to select the correct templater from the `spec.templater` in the [Template Entity](../../software-catalog/descriptor-format.md#kind-template). -If you wish to add a new templater you'll need to register it with the +If you wish to add a new templater, you'll need to register it with the `TemplaterBuilder`. ### Creating your own Templater to add to the `TemplaterBuilder` @@ -80,10 +84,10 @@ follows: - `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to be able to run docker containers. -_note_ currently the templaters that we provide are basically docker action +_note_ Currently the templaters that we provide are basically Docker action containers that are run on top of the skeleton folder. This keeps dependencies to a minimal for running backstage scaffolder, but you don't /have/ to use -docker. You could create your own templater that spins up an EC2 instance and +Docker. You could create your own templater that spins up an EC2 instance and downloads the folder and does everything using an AMI if you want. It's entirely up to you! @@ -113,8 +117,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: handlebars @@ -135,7 +139,7 @@ spec: description: Description of the component ``` -You see that the `spec.templater` is set as `handlebars`, you'll need to +You see that the `spec.templater` is set as `handlebars`, so you'll need to register this with the `TemplaterBuilder` like so: ```ts diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md index a55128c57d..02d27a5725 100644 --- a/docs/features/software-templates/extending/index.md +++ b/docs/features/software-templates/extending/index.md @@ -1,12 +1,15 @@ -## Extending the Scaffolder +--- +id: extending-index +title: Extending the Scaffolder +--- Welcome. Take a seat. You're at the Scaffolder Documentation. -So - You wanna create stuff inside your company from some prebaked templates? +So, you want to create stuff inside your company from some prebaked templates? You're at the right place. -This guide is gonna take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through whats going on in the backend to be +This guide is going to take you through how the Scaffolder in Backstage works. +We'll dive into some jargon and run through what's going on in the backend to be able to create these templates. There's also more guides that you might find useful at the bottom of this document. At it's core, theres 3 simple stages. @@ -22,9 +25,9 @@ scaffolder that you will need to know: 3. Publish Each of these steps can be configured for your own use case, but we provide some -sensible defaults too. +sensible defaults, too. -Lets dive a little deeper into these phases. +Let's dive a little deeper into these phases. ### Glossary and Jargon @@ -35,8 +38,8 @@ the router to pick the correct `Preparer` to run for the `Template` entity. **Templater** - The templater is responsible for actually running the chosen templater on top of the previously returned temporary directory from the -**Preprarer**. We advise making these docker containers as it can keep all -dependencies, for example Cookiecutter, self contained and not a dependency on +**Preprarer**. We advise making these Docker containers as it can keep all +dependencies--for example Cookiecutter--self contained and not a dependency on the host machine. **Publisher** - The publisher is responsible for taking the finished directory, @@ -47,11 +50,11 @@ passed through to the scaffolder backend. ### How it works -The main of the heavy lifting is done in the +Most of the heavy lifting is done in the [router.ts](https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) file in the `scaffolder-backend` plugin. -There are 2 routes defined in the router. `POST /v1/jobs` and +There are two routes defined in the router: `POST /v1/jobs` and `GET /v1/job/:jobId` To create a scaffolding job, a JSON object containing the @@ -75,7 +78,7 @@ additional templating values must be posted as the post body. The values should represent something that is valid with the `schema` part of the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) -Once that has been posted, a job will be setup with different stages. And the +Once that has been posted, a job will be setup with different stages, and the job processor will complete each stage before moving onto the next stage, whilst collecting logs and mutating the running job. diff --git a/docs/features/software-templates/index.md b/docs/features/software-templates/index.md index c7358518cd..ae121f8029 100644 --- a/docs/features/software-templates/index.md +++ b/docs/features/software-templates/index.md @@ -1,19 +1,33 @@ -# Software Templates +--- +id: software-templates-index +title: Backstage Software Templates +sidebar_label: Overview +description: The Software Templates part of Backstage is a tool that can help +you create Components inside Backstage +--- The Software Templates part of Backstage is a tool that can help you create -Components inside Backstage. It by default has the ability to load skeletons of -code, template in some variables and then publish the template to some location -like GitHub. +Components inside Backstage. By default, it has the ability to load skeletons of +code, template in some variables, and then publish the template to some +locations like GitHub or GitLab. + + ### Getting Started -The Software Templates are available under `/create`, and if you've followed -[Getting Started with Backstage](../../getting-started), you should be able to -reach `http://localhost:3000/create`. +> Be sure to have covered [Installing in your Backstage App](./installation.md) +> for your separate App or +> [Getting Started with Backstage](../../getting-started) for this repo before +> proceeding. -You should get something that looks similar to this: +The Software Templates are available under `/create`. For local development you +should be able to reach them at `http://localhost:3000/create`. -![Create Image](./assets/create.png) +Once there, you should see something that looks similar to this: + +![Create Image](../../assets/software-templates/create.png) ### Choose a template @@ -22,38 +36,40 @@ page which may or may not look different for each template. Each template can ask for different input variables, and they are then passed to the templater internally. -![Enter some variables](./assets/template-picked.png) +![Enter some variables](../../assets/software-templates/template-picked.png) After filling in these variables, you'll get some more fields to fill out which -are required for backstage usage. The owner, which is a `user` in the backstage -system, and the `storePath` which right now must be a Github Organisation and a -non-existing github repository name in the format `organistaion/reponame`. +are required for backstage usage: the owner, (which is a `user` in the backstage +system), the `storePath` (which right now must be a GitHub Organisation or +GitHub user), a non-existing github repository name in the format +`organisation/reponame`, and a GitHub team or user account which should be +granted admin access to the repository. -![Enter backstage vars](./assets/template-picked-2.png) +![Enter backstage vars](../../assets/software-templates/template-picked-2.png) ### Run! Once you've entered values and confirmed, you'll then get a modal with live progress of what is currently happening with the creation of your template. -![Templating Running](./assets/running.png) +![Templating Running](../../assets/software-templates/running.png) It shouldn't take too long, and you'll have a success screen! -![Templating Complete](./assets/complete.png) +![Templating Complete](../../assets/software-templates/complete.png) If it fails, you'll be able to click on each section to get the log from the -step that failed which can be helpful to debug. +step that failed which can be helpful in debugging. -![Templating failed](./assets/failed.png) +![Templating failed](../../assets/software-templates/failed.png) ### View Component in Catalog -When it's been created you'll see the `View in Catalog` button, which will take +When it's been created, you'll see the `View in Catalog` button, which will take you to the registered component in the catalog: -![Catalog](./assets/go-to-catalog.png) +![Catalog](../../assets/software-templates/go-to-catalog.png) And then you'll also be able to see it in the Catalog View table -![Catalog](./assets/added-to-the-catalog-list.png) +![Catalog](../../assets/software-templates/added-to-the-catalog-list.png) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md new file mode 100644 index 0000000000..5453c4befa --- /dev/null +++ b/docs/features/software-templates/installation.md @@ -0,0 +1,257 @@ +--- +id: installation +title: Installing in your Backstage App +description: Documentation on How to install Backstage App +--- + +The scaffolder plugin comes in two packages, `@backstage/plugin-scaffolder` and +`@backstage/plugin-scaffolder-backend`. Each has their own installation steps, +outlined below. + +The Scaffolder plugin also depends on the Software Catalog. Instructions for how +to set that up can be found [here](../software-catalog/installation.md). + +## Installing @backstage/plugin-scaffolder + +> **Note that if you used `npx @backstage/create-app`, the plugin may already be +> present** + +The scaffolder frontend plugin should be installed in your `app` package, which +is created as a part of `@backstage/create-app`. To install the package, run: + +```bash +cd packages/app +yarn add @backstage/plugin-scaffolder +``` + +Make sure the version of `@backstage/plugin-scaffolder` matches the version of +other `@backstage` packages. You can update it in `packages/app/package.json` if +it doesn't. + +### Adding the Plugin to your `packages/app` + +Add the following entry to the head of your `packages/app/src/plugins.ts`: + +```ts +export { plugin as ScaffolderPlugin } from '@backstage/plugin-scaffolder'; +``` + +Add the following to your `packages/app/src/apis.ts`: + +```ts +import { scaffolderApiRef, ScaffolderApi } from '@backstage/plugin-scaffolder'; + +// Inside the ApiRegistry builder function ... + +builder.add( + scaffolderApiRef, + new ScaffolderApi({ + apiOrigin: backendUrl, + basePath: '/scaffolder/v1', + }), +); +``` + +Where `backendUrl` is the `backend.baseUrl` from config, i.e. +`const backendUrl = config.getString('backend.baseUrl')`. + +This is all that is needed for the frontend part of the Scaffolder plugin to +work! + +## Installing @backstage/plugin-scaffolder-backend + +> **Note that if you used `npx @backstage/create-app`, the plugin may already be +> present** + +The scaffolder backend should be installed in your `backend` package, which is +created as a part of `@backstage/create-app`. To install the package, run: + +```bash +cd packages/backend +yarn add @backstage/plugin-scaffolder-backend +``` + +Make sure the version of `@backstage/plugin-scaffolder-backend` matches the +version of other `@backstage` packages. You can update it in +`packages/backend/package.json` if it doesn't. + +### Adding the Plugin to your `packages/backend` + +You'll need to add the plugin to the `backend`'s router. You can do this by +creating a file called `packages/backend/src/plugins/scaffolder.ts` with the +following contents to get you up and running quickly. + +```ts +import { + CookieCutter, + createRouter, + FilePreparer, + GithubPreparer, + GitlabPreparer, + Preparers, + Publishers, + GithubPublisher, + GitlabPublisher, + CreateReactAppTemplater, + Templaters, + RepoVisilityOptions, +} from '@backstage/plugin-scaffolder-backend'; +import { Octokit } from '@octokit/rest'; +import { Gitlab } from '@gitbeaker/node'; +import type { PluginEnvironment } from '../types'; +import Docker from 'dockerode'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + const cookiecutterTemplater = new CookieCutter(); + const craTemplater = new CreateReactAppTemplater(); + const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); + templaters.register('cra', craTemplater); + + const filePreparer = new FilePreparer(); + const githubPreparer = new GithubPreparer(); + const gitlabPreparer = new GitlabPreparer(config); + const preparers = new Preparers(); + + preparers.register('file', filePreparer); + preparers.register('github', githubPreparer); + preparers.register('gitlab', gitlabPreparer); + preparers.register('gitlab/api', gitlabPreparer); + + const publishers = new Publishers(); + + const githubToken = config.getString('scaffolder.github.token'); + const repoVisibility = config.getString( + 'scaffolder.github.visibility', + ) as RepoVisilityOptions; + + const githubClient = new Octokit({ auth: githubToken }); + const githubPublisher = new GithubPublisher({ + client: githubClient, + token: githubToken, + repoVisibility, + }); + publishers.register('file', githubPublisher); + publishers.register('github', githubPublisher); + + const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api'); + + if (gitLabConfig) { + const gitLabToken = gitLabConfig.getString('token'); + const gitLabClient = new Gitlab({ + host: gitLabConfig.getOptionalString('baseUrl'), + token: gitLabToken, + }); + const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken); + publishers.register('gitlab', gitLabPublisher); + publishers.register('gitlab/api', gitLabPublisher); + } + + const dockerClient = new Docker(); + return await createRouter({ + preparers, + templaters, + publishers, + logger, + dockerClient, + }); +} +``` + +Once the `scaffolder.ts` router setup file is in place, add the router to +`packages/backend/src/index.ts`: + +```ts +import scaffolder from './plugins/scaffolder'; + +const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + +const service = createServiceBuilder(module) + .loadConfig(configReader) + /** several different routers */ + .addRouter('/scaffolder', await scaffolder(scaffolderEnv)); +``` + +### Adding Templates + +At this point the scaffolder backend is installed in your backend package, but +you will not have any templates available to use. These need to be added to the +software catalog, as they are represented as entities of kind +[Template](../software-catalog/descriptor-format.md#kind-template). You can find +out more about adding templates [here](./adding-templates.md). + +To get up and running and try out some templates quickly, you can add some of +our example templates through static configuration. Add the following to the +`catalog.locations` section in your `app-config.yaml`: + +```yaml +catalog: + locations: + # Backstage Example Templates + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml + - type: github + target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml + - type: github + target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml +``` + +### Runtime Dependencies / Configuration + +For the scaffolder backend plugin to function, it needs a GitHub access token, +and access to a running Docker daemon. You can create a GitHub access token +[here](https://github.com/settings/tokens/new), select `repo` scope only. Full +docs on creating private GitHub access tokens is available +[here](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). +Note that the need for private GitHub access tokens will be replaced with GitHub +Apps integration further down the line. + +#### Github + +The Github access token is retrieved from environment variables via the config. +The config file needs to specify what environment variable the token is +retrieved from. Your config should have the following objects. + +#### Gitlab + +For Gitlab, we currently support the configuration of the GitLab publisher and +allows to configure the private access token and the base URL of a GitLab +instance: + +```yaml +scaffolder: + github: + token: + $secret: + env: GITHUB_ACCESS_TOKEN + visibility: public # or 'internal' or 'private' + gitlab: + api: + baseUrl: https://gitlab.com + token: + $secret: + env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN +``` + +You can configure who can see the new repositories that the scaffolder creates +by specifying `visibility` option. Valid options are `public`, `private` and +`internal`. `internal` options is for GitHub Enterprise clients, which means +public within the organization. + +### Running the Backend + +Finally, make sure you have a local Docker daemon running, and start up the +backend with the new configuration: + +```bash +cd packages/backend +GITHUB_ACCESS_TOKEN= yarn start +``` + +If you've also set up the frontend plugin, so you should be ready to go browse +the templates at [localhost:3000/create](http://localhost:3000/create) now! diff --git a/docs/features/techdocs/FAQ.md b/docs/features/techdocs/FAQ.md index ee9a40c983..5d6c9136d9 100644 --- a/docs/features/techdocs/FAQ.md +++ b/docs/features/techdocs/FAQ.md @@ -1,20 +1,26 @@ -# TechDocs FAQ +--- +id: faqs +title: TechDocs FAQ +sidebar_label: FAQ +description: This page answers frequently asked questions about TechDocs +--- -This page answer frequently asked questions about [TechDocs]. +This page answers frequently asked questions about [TechDocs](README.md). -#### Technology +## Technology -- [What static site generator is TechDocs using?](./#what-static-site-generator-is-techdocs-using) -- [What is the mkdocs-techdocs-core plugin?](./#what-is-the-mkdocs-techdocs-core-plugin) +- [What static site generator is TechDocs using?](#what-static-site-generator-is-techdocs-using) +- [What is the mkdocs-techdocs-core plugin?](#what-is-the-mkdocs-techdocs-core-plugin) +- [Does TechDocs support file formats other than Markdown (e.g. rst, asciidoc)?](#does-techdocs-support-file-formats-other-than-markdown-eg-rst-asciidoc-) -## What static site generator is TechDocs using? +#### What static site generator is TechDocs using? TechDocs is using [MkDocs](https://www.mkdocs.org/) to build project -doucmentation under the hood. Documentation built with the +documentation under the hood. Documentation built with the [techdocs-container](https://github.com/spotify/backstage/blob/master/packages/techdocs-container/README.md) is using the MkDocs Material Theme. -## What is the mkdocs-techdocs-core plugin? +#### What is the mkdocs-techdocs-core plugin? The [mkdocs-techdocs-core](https://github.com/spotify/backstage/blob/master/packages/techdocs-container/techdocs-core/README.md) @@ -23,7 +29,9 @@ plugins (e.g. [MkDocs Monorepo Plugin](https://github.com/spotify/mkdocs-monorepo-plugin)) as well as a selection of Python Markdown extensions that TechDocs supports. -_Add a question that you think others might be interested in? Edit the file -[here](https://github.com/spotify/backstage/edit/master/docs/features/techdocs/FAQ.md)._ +#### Does TechDocs support file formats other than Markdown (e.g. rst, asciidoc) ? -[techdocs]: README.md +Not right now. We are currently using MkDocs to generate the documentation from +source. So, they have to be in Markdown format. However, in future we want to +support other alternatives to MkDocs. That will make it possible to use other +file formats. diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index b6a4d82b92..aa97506edb 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -1,37 +1,85 @@ -# TechDocs Documentation +--- +id: techdocs-overview +title: TechDocs Documentation +sidebar_label: Overview +description: TechDocs is Spotify’s homegrown docs-like-code solution built +directly into Backstage +--- ## What is it? -Wait, what is TechDocs? TechDocs is Spotify’s homegrown docs-like-code solution -built directly into Backstage. Today, it is now one of the core products in -Spotify’s developer experience offering with 2,400+ documentation sites and -1,000+ engineers using it daily. +TechDocs is Spotify’s homegrown docs-like-code solution built directly into +Backstage. This means engineers write their documentation in Markdown files +which live together with their code. + +Today, it is one of the core products in Spotify’s developer experience offering +with 2,400+ documentation sites and 1,000+ engineers using it daily. Read more +about TechDocs and the philosophy in its +[announcement blog post](https://backstage.io/blog/2020/09/08/announcing-tech-docs). +🎉 ## Features -- A centralized place to discover documentation. +- A centralized place to discover and read documentation. -- A clear end-to-end docs-like-code solution. (_Coming soon in V.1_) +- A clear end-to-end docs-like-code solution. - A tightly coupled feedback loop with the developer workflow. (_Coming soon in - V.2_) + V.3_) -- A developer ecosystem for creating extensions. (_Coming soon in V.2_) +- A developer ecosystem for creating extensions. (_Coming soon in V.3_) ## Project roadmap -| Version | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| [TechDocs V.0 ✅][v0] | Read docs in Backstage - Enable anyone to get a reader experience working in Backstage. | -| [TechDocs V.1 🚧][v1] | TechDocs end to end - First and minimum release of TechDocs that you can use end to end - and contribute to. | -| [TechDocs V.2 🔮⌛][v2] | Widget Architecture - TechDocs widget architecture available, so the community can create their own customized features. | +| Version | Description | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| [TechDocs V.0 ✅][v0] | Read docs in Backstage - Enable anyone to get a reader experience working in Backstage. [See V.0 Use Cases.](#techdocs-v0) | +| [TechDocs V.1 ✅][v1] | TechDocs end to end (alpha) - Alpha of TechDocs that you can use end to end - and contribute to. [See V.1 Use Cases.](#techdocs-v1) | +| [TechDocs V.2 🔮⌛][v2] | Platform stability and compatibility improvements. [See V.2 Use Cases.](#techdocs-v2) | +| TechDocs V.3 🔮⌛ | Widget Architecture - TechDocs widget architecture available, so the community can create their own customized features. | [v0]: https://github.com/spotify/backstage/milestone/15 [v1]: https://github.com/spotify/backstage/milestone/16 [v2]: https://github.com/spotify/backstage/milestone/17 + + +## Use Cases + +#### TechDocs V.0 + +- As a user I can navigate to a manually curated docs explore page. +- As a user I can navigate to and read mock documentation that is manually + uploaded by the TechDocs core team. + +#### TechDocs V.1 + +- As a user I can run TechDocs locally and read documentation. +- As a user I can create a docs folder in my entity project and add a reference + in the entity configuration file (of the owning entity) to my documentation. + - Backstage will automatically build my documentation and serve it in + TechDocs. + - Documentation will be displayed under the docs tab in the service catalog. +- As a user I can create a docs only repository that will be standalone from any + other service. +- As a user I can choose my own storage solution for the documentation (as + example GCS/AWS/Azure etc) +- As a user I can define my own API to interface my own documentation solution. + +#### TechDocs V.2 + +Platform stability and compatibility improvements + +- As a user I can define the metadata generated for my documentation. +- As a user I will be able to browse metadata from within my documentation in + Backstage. + +#### TechDocs V.3 + +more to come... + ## Structure - [Getting Started] diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md new file mode 100644 index 0000000000..926ec238e1 --- /dev/null +++ b/docs/features/techdocs/architecture.md @@ -0,0 +1,7 @@ +--- +id: architecture +title: Architecture +description: Documentation on Architecture +--- + +![TechDocs Big Picture](../../assets/techdocs/techdocs_big_picture.png) diff --git a/docs/features/techdocs/concepts.md b/docs/features/techdocs/concepts.md index d0e3c3c0ea..f13f9933f5 100644 --- a/docs/features/techdocs/concepts.md +++ b/docs/features/techdocs/concepts.md @@ -1,15 +1,20 @@ -# Concepts +--- +id: concepts +title: Concepts +description: Documentation on concepts that are introduced with +Spotify's docs-like-code solution in Backstage +--- -This page describes concepts that has been introduced with Spotify's -docs-like-code solution in Backstage. +This page describes concepts that are introduced with Spotify's docs-like-code +solution in Backstage. ### TechDocs Core Plugin -The TechDocs Core Plugin is a MkDocs plugin created as a wrapper around multiple -MkDocs plugins and Python Markdown extensions to standardize the configuration -of MkDocs used for TechDocs. +The TechDocs Core Plugin is an [MkDocs](https://www.mkdocs.org/) plugin created +as a wrapper around multiple MkDocs plugins and Python Markdown extensions to +standardize the configuration of MkDocs used for TechDocs. -[TechDocs Core](../../../packages/techdocs-container/techdocs-core/README.md) +[TechDocs Core](https://github.com/spotify/backstage/blob/master/packages/techdocs-container/techdocs-core/README.md) ### TechDocs container @@ -18,36 +23,36 @@ The TechDocs container is a Docker container available at pages, including stylesheets and scripts from Python flavored Markdown, through MkDocs. -[TechDocs Container](../../../packages/techdocs-container/README.md) +[TechDocs Container](https://github.com/spotify/backstage/blob/master/packages/techdocs-container/README.md) -### TechDocs publisher (Coming Soon) +### TechDocs publisher (coming soon) ### TechDocs CLI The TechDocs CLI was created to make it easy to write, generate and preview documentation for publishing. Currently it mostly acts as a wrapper around the -TechDocs container and provides a easy to use interface for our docker +TechDocs container and provides an easy-to-use interface for our docker container. -[TechDocs CLI](../../../packages/techdocs-cli/README.md) +[TechDocs CLI](https://github.com/spotify/backstage/blob/master/packages/techdocs-cli/README.md) ### TechDocs Reader -Documentation generated by TechDocs is generated as static html sites. The -TechDocs Reader was therefore created to be able to integrate pre-generated html +Documentation generated by TechDocs is generated as static HTML sites. The +TechDocs Reader was therefore created to be able to integrate pre-generated HTML sites with the Backstage UI. The TechDocs Reader purpose is also to open up the opportunity to integrate TechDocs widgets for a customized full-featured TechDocs experience. -([Coming Soon V.2](https://github.com/spotify/backstage/milestone/17)) +([coming soon V.3](./README.md#project-roadmap)) -[TechDocs Reader](../../../plugins/techdocs/src/reader/README.md) +[TechDocs Reader](https://github.com/spotify/backstage/blob/master/plugins/techdocs/src/reader/README.md) ### Transformers -Transformers is different pieces of functionality used inside the TechDocs -Reader. The reason to why transformers were introduced is to provide a way to -transform the html content on pre and post render. (e.g. rewrite docs links or -modify css) +Transformers are different pieces of functionality used inside the TechDocs +Reader. The reason why transformers were introduced was to provide a way to +transform the HTML content on pre and post render (e.g. rewrite docs links or +modify css). -[Transformers API docs](../../../plugins/techdocs/src/reader/transformers/README.md) +[Transformers API docs](https://github.com/spotify/backstage/blob/master/plugins/techdocs/src/reader/README.md) diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 0ac4f054d7..e1adedf25e 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -1,28 +1,44 @@ -# Creating and publishing your docs +--- +id: creating-and-publishing +title: Creating and publishing your docs +sidebar_label: Creating and Publishing Documentation +description: Guidance on how to create and publish documentation +--- This section will guide you through: -- Creating a basic setup for your documentation -- Writing and previewing your documentation in a local Backstage environment -- Creating a build ready for publication -- Publishing your documentation and making your Backstage instance read from - your published docs. +- [Create a basic documentation setup](#create-a-basic-documentation-setup) + - [Use the documentation template](#use-the-documentation-template) + - [Manually add documentation setup to already existing repository](#manually-add-documentation-setup-to-already-existing-repository) +- [Writing and previewing your documentation](#writing-and-previewing-your-documentation) -## Prerequisities +## Prerequisites -- [Docker](https://docs.docker.com/get-docker/) -- Static file hosting -- A working Backstage instance with TechDocs installed - [TechDocs getting started](getting-started.md) +- A working Backstage instance with TechDocs installed (see + [TechDocs getting started](getting-started.md)) ## Create a basic documentation setup -Create a directory that contains your documentation. Inside this directory you -should create a file called `mkdocs.yml`. As an example you can create a -directory called `hello-docs` in your home directory (also known as `~`). Below -is a basic example of how it could look. +### Use the documentation template -The `~/hello-docs/mkdocs.yml` file should have the following content: +Your working Backstage instance should by default have a documentation template +added. If not, follow these +[instructions](../software-templates/installation.md#adding-templates) to add +the documentation template. + +![Documentation Template](../../assets/techdocs/documentation-template.png) + +Create an entity from the documentation template and you will get the needed +setup for free. + +### Manually add documentation setup to already existing repository + +Prerequisities: + +- `catalog-info.yml` file registered to Backstage. + +Create a `mkdocs.yml` file in the root of the repository with the following +content: ```yaml site_name: 'example-docs' @@ -34,7 +50,20 @@ plugins: - techdocs-core ``` -And then the `~/hello-docs/docs/index.md` should have the following content: +Update your `catalog-info.yaml` file in the root of the repository with the +following content: + +```yaml +metadata: + annotations: + backstage.io/techdocs-ref: dir:./ +``` + +Create a `/docs` folder in the root of the project with at least a `index.md` +file. _(If you add more markdown files, make sure to update the nav in the +mkdocs.yml file to get a proper navigation for your documentation.)_ + +The `docs/index.md` can for example have the following content: ```md # example docs @@ -42,93 +71,18 @@ And then the `~/hello-docs/docs/index.md` should have the following content: This is a basic example of documentation. ``` +Commit your changes, open a pull request and merge. You will now get your +updated documentation next time you run Backstage! + ## Writing and previewing your documentation Using the `techdocs-cli` you can preview your docs inside a local Backstage -instance and get automatic recompilation on changes. This is useful for when you -want to write your documentation. +instance and get live reload on changes. This is useful when you want to preview +your documentation while writing. To do this you can run: ```bash -cd ~/hello-docs/ -npx techdocs-cli serve +cd /path/to/docs-repository/ +npx @techdocs/cli serve ``` - -## Build production ready documentation - -To get a build suitable for publication you can build your docs using the -`spotify/techdocs` container. - -```bash -cd ~/hello-docs/ -docker run -it -w /content -v $(pwd):/content spotify/techdocs build -``` - -You should now have a folder called `~/hello-docs/site/`. - -## Deploy to a file server - -In order to serve documentation to TechDocs, our Backstage plugin needs to -download the HTML rendered from the -[Create documentation](#create-documentation) step above. This will likely exist -on an external file server, or a storage solution such as Google Cloud Storage. - -When deploying documentation, it should be deployed on that file server / -storage solution with the following convention: `{id}/{file}`. For example, if -we wanted to upload the `getting-started/index.html` file for the `backstage` -documentation site, we would upload it to our file server as -`backstage/getting-started/index.html`. - -To explain further how this would look like for multiple documentation sites, -take a look at this example file tree that would be represented on your file -server: - -```md -/backstage/index.html /backstage/getting-started/index.html -/backstage/contributing/index.html /mkdocs/index.html -/mkdocs/plugin-development/index.html -/mkdocs/plugin-development/debugging/index.html -``` - -In this file tree, we have two documentation sites available: `backstage` and -`mkdocs`. Each of them expose several pages. Let's say both of these are hosted -on `http://example.com` as the server URL. - -When you configure the TechDocs plugin in Backstage to use `http://example.com` -as the file server / storage solution, it will translate the following URLs to -the file server: - -| Backstage URL | File Server URL | -| --------------------------------------------------------- | ------------------------------------------------------- | -| https://demo.backstage.io/docs/backstage/ | http://example.com/backstage/index.html | -| https://demo.backstage.io/docs/mkdocs/plugin-development/ | http://example.com/mkdocs/plugin-development/index.html | - -Then deploying new sites is easy. It's as simple as copying over the `site/` -folder produced in the [Create documentation](#create-documentation) step above -and copying it over to the file server / storage solution under the ID of the -documentation site. It will then become immediately available in Backstage under -the same ID as you can see in the table above. - -So, if the URL to your file server is `http://example.com/`, your -`~/hello-docs/site` folder containing the documentation should be accessible at -`http://example.com/hello-docs/`. - -## Configure TechDocs to read from file server - -In order for Backstage to show your documentation, it needs to know where you -uploaded it. - -Make sure you have Backstage set up using -[TechDocs getting started](getting-started.md) - -To point Backstage to your docs storage, add or change the following lines in -your Backstage `app-config.yaml`: - -```yaml -techdocs: - storageUrl: http://example.com -``` - -You can now start Backstage using `yarn start` and open up your browser at -`http://localhost:3000/docs/hello-docs` to view your docs. diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index 2ea8fc7fa9..b9d215fd8a 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -1,76 +1,39 @@ -# Getting Started +--- +id: getting-started +title: Getting Started +description: Getting Started Guidelines +--- -> TechDocs is not feature complete and currently you can't set up a complete -> end-to-end working TechDocs plugin without customizing the plugin itself. +TechDocs functions as a plugin to Backstage, so you will need to use Backstage +to use TechDocs. -> With TechDocs V.0 you can expect a demonstration of how to integrate docs into -> Backstage. Currently it can create docs using -> [mkdocs](https://www.mkdocs.org/), as well as reading published docs. If you -> publish generated docs and passing in a storageUrl in your `app-config.yaml` -> you can view it in Backstage by going to -> `http://localhost:3000/docs/`. - -Getting started with TechDocs is easy. TechDocs functions as a plugin to -Backstage, why you will need to use Backstage to use TechDocs. - -## What is Backstage? - -Backstage is an open platform for building developer portals. It’s based on the -developer portal we’ve been using internally at Spotify for over four years. -[Read more](https://github.com/spotify/backstage). - -## Prerequisities - -In order to use Backstage and TechDocs, you will need to have the following -installed: - -- [Node.js](https://nodejs.org) Active LTS (long term support), currently v12 -- [Yarn](https://yarnpkg.com/getting-started/install) - -## Creating a new Backstage app - -> If you have already created a Backstage application for this purpose, jump to -> [Installing TechDocs](#installing-techdocs), otherwise complete this step. - -To create a new Backstage application for us to set up TechDocs, you will need -to run the following command: - -```bash -npx @backstage/cli create-app -``` - -You will then be prompted to enter a name for your application. Once you do so, -this will create a new Backstage application for you in a new folder. For -example, if we chose the name `hello-world` for our application, it would create -a new `hello-world` folder containing our new Backstage application. +If you haven't setup Backstage already, start +[here](../../getting-started/index.md). ## Installing TechDocs -Inside of our new Backstage application, TechDocs is not provided by default. -For this reason we will need to manually set up TechDocs. It should take less -than a minute. +TechDocs is provided with the Backstage application by default. If you want to +set up TechDocs manually, keep follow the instructions below. ### Adding the package -We will need to add our plugin to your Backstage application. To do so, you can -navigate to your new Backstage application folder and then run a single command -to install TechDocs. +The first step is to add the TechDocs plugin to your Backstage application. +Navigate to your new Backstage application folder: ```bash cd hello-world/ ``` -Then you need to navigate to your `packages/app` folder to install TechDocs: +Then navigate to your `packages/app` folder to install TechDocs: ```bash cd packages/app yarn add @backstage/plugin-techdocs ``` -After a short while, it should successfully install the TechDocs plugin. Now we -just need to set up some basic configuration! +After a short while, the TechDocs plugin should be successfully installed. -Enter the following command: +Next, you need to set up some basic configuration. Enter the following command: ```bash yarn install @@ -85,25 +48,72 @@ export { plugin as TechDocs } from '@backstage/plugin-techdocs'; ### Setting the configuration TechDocs allows for configuration of the docs storage URL through your -app-config file. The URL provided here is demo docs used to testing. +`app-config.yaml` file. We provide two different values to be configured, +`requestUrl` and `storageUrl`. The `requestUrl` is what the reader will request +its data from, and `storageUrl` is where the backend can find the stored +documentation. -To use the demo docs, add the following lines to `app-config.yaml`: +The default storage and request URLs: ```yaml techdocs: - storageUrl: https://techdocs-mock-sites.storage.googleapis.com + storageUrl: http://localhost:7000/techdocs/static/docs + requestUrl: http://localhost:7000/techdocs/docs ``` -## Run Backstage Locally +If you want `techdocs-backend` to manage building and publishing, you want +`requestUrl` to point to the default value (or wherever `techdocs-backend` is +hosted). `storageUrl` should be where your publisher publishes your docs. Using +the default `LocalPublish` that is the default value. -Change folder to your Backstage application root. +If you have a setup where you are not using `techdocs-backend` for managing +building and publishing of your documentation, you want to change the +`requestUrl` to point to your storage. In this case `storageUrl` is not +required. + +### Disable Docker in Docker situation (Optional) + +The TechDocs backend plugin runs a docker container with mkdocs to generate the +frontend of the docs from source files (Markdown). If you are deploying +Backstage using Docker, this will mean that your Backstage Docker container will +try to run another Docker container for TechDocs backend. + +To avoid this problem, we have a configuration available. You can set a value in +your `app-config.yaml` that tells the techdocs generator if it should run the +`local` mkdocs or run it from `docker`. This defaults to running as `docker` if +no config is provided. + +```yaml +techdocs: + generators: + techdocs: local +``` + +Setting `generators.techdocs` to `local` means you will have to make sure your +environment is compatible with techdocs. You will have to install the +`mkdocs-techdocs-container` and 'mkdocs' package from pip, as well as graphviz +and plantuml from your package manager. This has only been tested with python +3.7 and python 3.8. + +## Run Backstage locally + +Change folder to `/packages/backend` and run the +following command: ```bash yarn start ``` -Open browser at [http://localhost:3000/docs/](http://localhost:3000/docs/) +Open a new command line window. Change directory to your Backstage application +root and run the following command: -## Extra Reading +```bash +yarn start +``` -[Back to Docs](README.md) +Open your browser at [http://localhost:3000/docs/](http://localhost:3000/docs/). + +## Additional reading + +- [Creating and publishing your docs](creating-and-publishing.md) +- [Back to README](README.md) diff --git a/docs/getting-started/app-custom-theme.md b/docs/getting-started/app-custom-theme.md index 8c8b150171..97876c2ba1 100644 --- a/docs/getting-started/app-custom-theme.md +++ b/docs/getting-started/app-custom-theme.md @@ -1,4 +1,8 @@ -# Custom App Themes +--- +id: app-custom-theme +title: Customize the look-and-feel of your App +description: Documentation on Customizing look and feel of the App +--- Backstage ships with a default theme with a light and dark mode variant. The themes are provided as a part of the diff --git a/docs/getting-started/configure-app-with-plugins.md b/docs/getting-started/configure-app-with-plugins.md index e69de29bb2..1d02f147c1 100644 --- a/docs/getting-started/configure-app-with-plugins.md +++ b/docs/getting-started/configure-app-with-plugins.md @@ -0,0 +1,37 @@ +--- +id: configure-app-with-plugins +title: Configuring App with plugins +description: Documentation on How Configuring App with plugins +--- + +## Adding existing plugins to your app + +Coming soon! + +### Adding a plugin page to the Sidebar + +In a standard Backstage app created with +[@backstage/create-app](./create-an-app.md), the sidebar is managed inside +`packages/app/src/sidebar.tsx`. The file exports the entire `Sidebar` element of +your app, which you can extend with additional entries by adding new +`SidebarItem` elements. + +For example, if you install the `api-docs` plugin, a matching `SidebarItem` +could be something like this: + +```tsx +// Import icon from MUI +import ExtensionIcon from '@material-ui/icons/Extension'; + +// ... inside the AppSidebar component +; +``` + +You can also use your own SVGs directly as icon components. Just make sure they +are sized according to the Material UI's +[SvgIcon](https://material-ui.com/api/svg-icon/) default of 24x24px, and set the +extension to `.icon.svg`. For example: + +```ts +import InternalToolIcon from './internal-tool.icon.svg'; +``` diff --git a/docs/getting-started/create-an-app.md b/docs/getting-started/create-an-app.md index b69ff1508c..452e90add5 100644 --- a/docs/getting-started/create-an-app.md +++ b/docs/getting-started/create-an-app.md @@ -1,4 +1,8 @@ -# Backstage App +--- +id: create-an-app +title: Create an App +description: Documentation on Creating an App +--- To get set up quickly with your own Backstage project you can create a Backstage App. @@ -12,60 +16,61 @@ To create a Backstage app, you will need to have [NodeJS](https://nodejs.org/en/download/) Active LTS Release installed (currently v12). -With `npx`: +Backstage provides a utility for creating new apps. It guides you through the +initial setup of selecting the name of the app and a database for the backend. +The database options are either SQLite or PostgreSQL, where the latter requires +you to set up a separate database instance. If in doubt, choose SQLite, but +don't worry about the choice, it's easy to change later! + +The easiest way to run the create app package is with `npx`: ```bash -npx @backstage/cli create-app +npx @backstage/create-app ``` This will create a new Backstage App inside the current folder. The name of the app-folder is the name that was provided when prompted.

- create app + create app

Inside that directory, it will generate all the files and folder structure needed for you to run your app. -### Folder structure +### General folder structure + +Below is a simplified layout of the files and folders generated when creating an +app. ``` app -├── README.md +├── app-config.yaml ├── lerna.json ├── package.json -├── prettier.config.js -├── tsconfig.json -├── packages -│ └── app -│ ├── package.json -│ ├── tsconfig.json -│ ├── public -│ │ └── ... -│ └── src -│ ├── App.test.tsx -│ ├── App.tsx -│ ├── index.tsx -│ ├── plugins.ts -│ └── setupTests.ts -└── plugins - └── welcome - ├── README.md - ├── package.json - ├── tsconfig.json - └── src - ├── index.ts - ├── plugin.test.ts - ├── plugin.ts - ├── setupTests.ts - └── components - ├── Timer - │ └── ... - └── WelcomePage - └── ... +└── packages +   ├── app +   └── backend ``` +- **app-config.yaml**: Main configuration file for the app. See + [Configuration](https://backstage.io/docs/conf/) for more information. +- **lerna.json**: Contains information about workspaces and other lerna + configuration needed for the monorepo setup. +- **package.json**: Root package.json for the project. _Note: Be sure that you + don't add any npm dependencies here as they probably should be installed in + the intended workspace rather than in the root._ +- **packages/**: Lerna leaf packages or "workspaces". Everything here is going + to be a separate package, managed by lerna. +- **packages/app/**: An fully functioning Backstage frontend app, that acts as a + good starting point for you to get to know Backstage. +- **packages/backend/**: We include a backend that helps power features such as + [Authentication](https://backstage.io/docs/auth/), + [Software Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview), + [Software Templates](https://backstage.io/docs/features/software-templates/software-templates-index) + and [TechDocs](https://backstage.io/docs/features/techdocs/techdocs-overview) + amongst other things. + ## Run the app When the installation is complete you can open the app folder and start the app. @@ -77,3 +82,13 @@ yarn start _When `yarn start` is ready it should open up a browser window displaying your app, if not you can navigate to `http://localhost:3000`._ + +In most cases you will want to start the backend as well, as it is required for +the catalog to work, along with many other plugins. + +To start the backend, open a separate terminal session and run the following in +the root directory: + +```bash +yarn workspace backend start +``` diff --git a/docs/getting-started/deployment-k8s.md b/docs/getting-started/deployment-k8s.md index e69de29bb2..ddcbd290c6 100644 --- a/docs/getting-started/deployment-k8s.md +++ b/docs/getting-started/deployment-k8s.md @@ -0,0 +1,7 @@ +--- +id: deployment-k8s +title: Kubernetes +description: Documentation on Kubernetes and K8s Deployment +--- + +Coming soon! diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 26c70acac4..2cd9708e2a 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -1,4 +1,8 @@ -# Deployment (Other) +--- +id: deployment-other +title: Other +description: Documentation on different ways of Deployment +--- ## Deploying Locally @@ -7,21 +11,20 @@ Run the following commands if you have Docker environment ```bash +$ yarn install $ yarn docker-build -$ docker run --rm -it -p 80:80 spotify/backstage +$ docker run --rm -it -p 7000:7000 -e NODE_ENV=development example-backend:latest ``` Then open http://localhost/ on your browser. ### Running with `docker-compose` -Run the following commands if you have docker and docker-compose for a full -example, with the example backend also deployed. +There is also a `docker-compose.yaml` that you can use to replace the previous +`docker run` command: ```bash -$ yarn docker-build:all +$ yarn install +$ yarn docker-build $ docker-compose up ``` - -Then open http://localhost:3000 on your browser to see the example app with an -example backend. diff --git a/docs/getting-started/development-environment.md b/docs/getting-started/development-environment.md index 02729e5bb4..154a959f85 100644 --- a/docs/getting-started/development-environment.md +++ b/docs/getting-started/development-environment.md @@ -1,4 +1,9 @@ -# Development Environment +--- +id: development-environment +title: Development Environment +description: Documentation on how to get set up for doing development on +the Backstage repository +--- This section describes how to get set up for doing development on the Backstage repository. @@ -62,6 +67,7 @@ yarn storybook # Start local storybook, useful for working on components in @bac yarn workspace @backstage/plugin-welcome start # Serve welcome plugin only, also supports --check yarn tsc # Run typecheck, use --watch for watch mode +yarn tsc:full # Run full type checking, for example without skipLibCheck, use in CI yarn build # Build published versions of packages, depends on tsc @@ -74,8 +80,6 @@ yarn test:all # test all packages yarn clean # Remove all output folders and @backstage/cli cache -yarn bundle # Build a production bundle of the example app - yarn diff # Make sure all plugins are up to date with the latest plugin template yarn create-plugin # Create a new plugin @@ -84,7 +88,3 @@ yarn create-plugin # Create a new plugin > See > [package.json](https://github.com/spotify/backstage/blob/master/package.json) > for other yarn commands/options. - -[Next Step - Create a Backstage plugin](../plugins/create-a-plugin.md) - -[Back to Docs](../README.md) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 2770b7a154..a9ac770f90 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -1,69 +1,51 @@ -# Getting started with Backstage +--- +id: index +title: Getting Started +description: Documentation on How to get started with Backstage +--- -## Running Backstage Locally +There are two different ways to get started with Backstage, either by creating a +standalone app, or by cloning this repo. Which method you use depends on what +you're planning to do. -To get up and running with a local Backstage to evaluate it, let's clone it off -of GitHub and run an initial build. First make sure that you have at least node -version 12 installed locally. +Creating a standalone instance makes it simpler to customize the application for +your needs whilst staying up to date with the project. You will also depend on +`@backstage` packages from NPM, making the project much smaller. This is the +recommended approach if you want to kick the tyres of Backstage or setup your +own instance. + +On the other hand, if you want to contribute plugins or to the project in +general, it's easier to fork and clone this project. That will let you stay up +to date with the latest changes, and gives you an easier path to make Pull +Requests towards this repo. + +### Creating a Standalone App + +Backstage provides the `@backstage/create-app` package to scaffold standalone +instances of Backstage. You will need to have +[NodeJS](https://nodejs.org/en/download/) Active LTS Release installed +(currently v12), and [yarn](https://classic.yarnpkg.com/en/docs/install). You +will also need to have [Docker](https://docs.docker.com/engine/install/) +installed to use some features like Software Templates and TechDocs. + +Using `npx` you can then run the following to create an app in a chosen +subdirectory of your current working directory: ```bash -# Start from your local development folder -git clone git@github.com:spotify/backstage.git -cd backstage - -# Fetch our dependencies and run an initial build -yarn install -yarn tsc -yarn build +npx @backstage/create-app ``` -Phew! Now you have a local repository that's ready to run and to add any open -source contributions into. +You will be taken through a wizard to create your app, and the output should +look something like this. You can read more about this process +[here](https://backstage.io/docs/getting-started/create-an-app). -We are now going to launch two things: an example Backstage frontend app, and an -example Backstage backend that the frontend talks to. You are going to need two -terminal windows, both starting from the Backstage project root. +### Contributing to Backstage -In the first window, run +You can read more in our +[CONTRIBUTING](https://github.com/spotify/backstage/blob/master/CONTRIBUTING.md) +guide, which can help you get setup with a Backstage development environment. -```bash -cd packages/backend -yarn start -``` +### Next steps -That starts up a backend instance on port 7000. - -In the other window, we will first populate the catalog with some nice mock data -to look at, and then launch the frontend. These commands are run from the -project root, not inside the backend directory. - -```bash -yarn lerna run mock-data -yarn start -``` - -That starts up the frontend on port 3000, and should automatically open a -browser window showing it. - -Congratulations! That should be it. Let us know how it went -[on discord](https://discord.gg/EBHEGzX), file issues for any -[feature](https://github.com/spotify/backstage/issues/new?labels=help+wanted&template=feature_template.md) -or -[plugin suggestions](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME), -or -[bugs](https://github.com/spotify/backstage/issues/new?labels=bug&template=bug_template.md) -you have, and feel free to -[contribute](https://github.com/spotify/backstage/blob/master/CONTRIBUTING.md)! - -## Creating a Plugin - -The value of Backstage grows with every new plugin that gets added. Here is a -collection of tutorials that will guide you through setting up and extending an -instance of Backstage with your own plugins. - -- [Development Environment](development-environment.md) -- [Create a Backstage Plugin](../plugins/create-a-plugin.md) -- [Structure of a Plugin](../plugins/structure-of-a-plugin.md) -- [Utility APIs](../api/utility-apis.md) - -[Back to Docs](../README.md) +Take a look at the [Running Backstage Locally](./running-backstage-locally.md) +guide to learn how to set up Backstage, and how to develop on the platform. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index e69de29bb2..3b11d2b8e1 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -0,0 +1,7 @@ +--- +id: installation +title: Installation +description: Documentation on Installation +--- + +Coming soon! diff --git a/docs/getting-started/running-backstage-locally.md b/docs/getting-started/running-backstage-locally.md new file mode 100644 index 0000000000..a0cf10998e --- /dev/null +++ b/docs/getting-started/running-backstage-locally.md @@ -0,0 +1,110 @@ +--- +id: running-backstage-locally +title: Running Backstage Locally +description: Documentation on How to run Backstage Locally +--- + +## Prerequisites + +- Node.js + +First make sure you are using NodeJS with an Active LTS Release, currently v12. +This is made easy with a version manager such as +[nvm](https://github.com/nvm-sh/nvm) which allows for version switching. + +```bash +# Installing a new version +nvm install 12 +> Downloading and installing node v12.18.3... +> Now using node v12.18.3 (npm v6.14.6) + +# Checking your version +node --version +> v12.18.3 +``` + +- yarn + +Please refer to the +[installation instructions for yarn](https://classic.yarnpkg.com/en/docs/install/). + +- Docker + +We use Docker for few of our core features. So, you will need Docker installed +locally to use features like Software Templates and TechDocs. Please refer to +the +[installation instructions for Docker](https://docs.docker.com/engine/install/). + +## Clone and Build + +To get up and running with a local Backstage to evaluate it, let's clone it off +of GitHub and run an initial build. + +```bash +# Start from your local development folder +git clone git@github.com:spotify/backstage.git +cd backstage + +# Fetch our dependencies and run an initial build +yarn install +yarn tsc +yarn build +``` + +Phew! Now you have a local repository that's ready to run and to add any open +source contributions into. + +We are now going to launch two things: an example Backstage frontend app, and an +example Backstage backend that the frontend talks to. You are going to need two +terminal windows, both starting from the Backstage project root. + +In the first window, run + +```bash +cd packages/backend +yarn start +``` + +That starts up a backend instance on port 7000. + +In the other window, we will then launch the frontend. This command is run from +the project root, not inside the backend directory. + +```bash +yarn start +``` + +That starts up the frontend on port 3000, and should automatically open a +browser window showing it. + +## Authentication + +When Backstage starts, you can choose to enter as a Guest user and start +exploring. + +But you can also set up any of the available authentication methods. The easiest +option will be GitHub. To setup GitHub authentication in Backstage, see +[these instructions](https://github.com/spotify/backstage/tree/master/plugins/auth-backend#github). + +--- + +Congratulations! That should be it. Let us know how it went +[on discord](https://discord.gg/EBHEGzX), file issues for any +[feature](https://github.com/spotify/backstage/issues/new?labels=help+wanted&template=feature_template.md) +or +[plugin suggestions](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME), +or +[bugs](https://github.com/spotify/backstage/issues/new?labels=bug&template=bug_template.md) +you have, and feel free to +[contribute](https://github.com/spotify/backstage/blob/master/CONTRIBUTING.md)! + +## Creating a Plugin + +The value of Backstage grows with every new plugin that gets added. Here is a +collection of tutorials that will guide you through setting up and extending an +instance of Backstage with your own plugins. + +- [Development Environment](development-environment.md) +- [Create a Backstage Plugin](../plugins/create-a-plugin.md) +- [Structure of a Plugin](../plugins/structure-of-a-plugin.md) +- [Utility APIs](../api/utility-apis.md) diff --git a/docs/overview/adopting.md b/docs/overview/adopting.md new file mode 100644 index 0000000000..422adfb288 --- /dev/null +++ b/docs/overview/adopting.md @@ -0,0 +1,162 @@ +--- +id: adopting +title: Strategies for adopting +description: Documentation on some general best practices that have been key +to Backstage's success inside Spotify +--- + +This document outlines some general best practices that have been key to +Backstage's success inside Spotify. Every organization is different and some of +these learnings will therefore not be applicable for your company. We are hoping +that this can become a living document, and strongly encourage you to contribute +back whatever learnings you gather while adopting Backstage inside your company. + +## Organizational setup + +The true value of Backstage is unlocked when it becomes _THE_ developer portal +at your company. As such it is important to recognize that you will need a +central team that owns your Backstage deployment and treats it like a product. + +This team will have **four** primary objectives: + +1. Maintain and operate your deployment of Backstage. This includes customer + support, infrastructure, CI/CD and, as your Backstage product grows, on-call + support. + +2. Drive adoption of customers (developers at your company). + +3. Work with senior tech leadership and architects to ensure your organizations + best practices for software development are encoded into a set of + [Software Templates](../features/software-templates/index.md). + +4. Evangelize Backstage as a central platform towards other + infrastructure/platform teams. + +## Internal evangelization + +The last objective deserves more attention, since it is the least obvious, but +also the most critical to successfully creating a consolidated platform. When +done right, Backstage acts as a "platform of platforms" or marketplace between +infra/platform teams and end-users: + +![pop](../assets/pop.png) + +While anyone at your company can contribute to the platform, the vast majority +of work will be done by teams that also has internal engineers as their +customers. The central team should treat these _contributing teams_ as customers +of the platform as well. + +These teams should be able to autonomously deliver value directly to their +customers. This is done primarily by building [plugins](../plugins/index.md). +Contributing teams should themselves treat their plugins as, or part of, the +products they maintain. + +> Case study: Inside Spotify we have a team that owns our CI platform. They +> don't only maintain the pipelines and build servers, but also expose their +> product in Backstage through a plugin. Since they also +> [maintain their own API](../plugins/call-existing-api.md), they can improve +> their product by iterating on API and UI in lockstep. Because the plugin +> follows our [platform design guidelines](../dls/design.md) their customers get +> a CI experience that is consistent with other tools on the platform (and users +> don't have to become experts in Jenkins). + +### Tactics + +Example of tactics we have used to evangelize Backstage internally: + +- Arrange "Lunch & Learns" and seminars. Frequently offer teams interested in + Backstage development to come to a seminar where you show, for example, how to + build a plugin from scratch. + +- Embedding. As contributing teams start development of their first plugin it is + often very appreciated to have one person from the central team come over and + "embed" for a Sprint or two. + +- Hack days. Backstage-focused Hackathons or hack days is a fun way to get + people into plugin development. + +- Show & tell meetings. In order to build an internal community around Backstage + we have quarterly meetings where anyone working on Backstage is invited to + present their work. This is a not only a great way to get early feedback, but + also helps coordination between teams that are building overlapping + experiences. + +- Provide metrics. Add instrumentation to your Backstage deployment and make + metrics available to contributing teams. At Spotify we have even gone so far + as sending out weekly digest email showing how usage metrics have changed for + individual plugins. + +- Pro-actively identify new plugins. Reach out to teams that own internal UIs or + platforms that you think would make sense to consolidate into Backstage. + +## KPIs and metrics + +These are some of the metrics that you can use to verify if Backstage has a +successful impact on your software development process: + +- **Onboarding time** Time until new engineers are productive. At Spotify we + measure this as the time until the employee has merged their 10th PR (this + metric was down 55% two years after deploying Backstage). Even though you may + not be onboarding engineers at a rapid pace, this metric is a great proxy for + the overall complexity of your ecosystem. Reducing it will therefore benefit + your whole engineering organization, not just new joiners. + +- **Number of merges per developer/day** Less time spent jumping between + different tools and looking for information means more time to focus on + shipping code. A second level of bottlenecks can be identified if you + categorize contributions by domain (services, web, data, etc). + +- **Deploys to production** Cousin to the metric above: How many times does an + engineer push changes into production. + +- **MTTR** With clear ownership of all the pieces in your micro services + ecosystem and all tools integrated into one place, Backstage makes it quicker + for teams to find the root cause of failures, and fix them. + +- **Context switching** Reducing context switching can help engineers stay in + the "zone". We measure the number of different tools an engineer have to + interact with in order to get a certain job done (e.g. push a change, follow + it into production and validate it did not break anything). + +- **T-shapedness** A + [T-shaped](https://medium.com/@jchyip/why-t-shaped-people-e8706198e437) + engineer is someone that is able to contribute to different domains of + engineering. Teams with T-shaped people have fewer bottlenecks and can + therefore deliver more consistently. Backstage makes it easier to be T-shaped + since tools and infrastructure is consistent between domains, and information + is available centrally. + +- **eNPS** Surveys asking about how productive people feel, how easy it is to + find information and overall satisfaction with internal tools. + +- **Fragmentation** _(Experimental)_ Backstage + [Software Templates](../features/software-templates/index.md) helps drive + standardization in your software ecosystem. By measuring the variance in + technology between different software components it is possible to get a sense + of the overall fragmentation in your ecosystem. Examples could include: + framework versions, languages, deployment methods and various code quality + measurements. + +Additionally, these proxy metrics can be used to validate the success of +Backstage as _the_ platform: + +- Nr of teams that have contributed at least one plugin (currently 63 inside + Spotify) + +- Nr of total plugins (currently 135 inside Spotify) + +- % of contributions coming from outside the central Backstage team (currently + 85% inside Spotify) + +- Traditional metrics such as visits (MAU, DAU, etc) and page views. Currently + ~50% of all Spotifiers use Backstage on a monthly basis, even though the + percentage of engineers is below 50%. Most engineers actually use Backstage on + a daily basis. + +Again, any feedback is appreciated. Please use the Edit button at the top of the +page to make a suggestion. + +_**Note!** It might be tempting to try to optimize Backstage usage and +"engagement". Even though you want to consolidate all your tooling and technical +documentation in Backstage, it is important to remember that time spent in +Backstage is time not spent writing code_ 🙃 diff --git a/docs/overview/architecture-overview.md b/docs/overview/architecture-overview.md index 9093e98e2e..f2c6dad61f 100644 --- a/docs/overview/architecture-overview.md +++ b/docs/overview/architecture-overview.md @@ -1,4 +1,10 @@ -# Typical Backstage architecture +--- +id: architecture-overview +title: Architecture overview +description: Documentation on Architecture overview +--- + +## Overview The following diagram shows how Backstage might look when deployed inside a company which uses the Tech Radar plugin, the Lighthouse plugin, the Circle CI @@ -14,27 +20,27 @@ Running this architecture in a real environment typically involves containerising the components. Various commands are provided for accomplishing this. -![The architecture of a basic Backstage application](./architecture-overview/backstage-typical-architecture.png) +![The architecture of a basic Backstage application](../assets/architecture-overview/backstage-typical-architecture.png) -# The UI +## The UI The UI is a thin, client-side wrapper around a set of plugins. It provides some core UI components and libraries for shared activities such as config management. [[live demo](https://backstage-demo.roadie.io/)] -![UI with different components highlighted](./architecture-overview/core-vs-plugin-components-highlighted.png) +![UI with different components highlighted](../assets/architecture-overview/core-vs-plugin-components-highlighted.png) Each plugin typically makes itself available in the UI on a dedicated URL. For example, the lighthouse plugin is registered with the UI on `/lighthouse`. [[live demo](https://backstage-demo.roadie.io/lighthouse)] -![The lighthouse plugin UI](./architecture-overview/lighthouse-plugin.png) +![The lighthouse plugin UI](../assets/architecture-overview/lighthouse-plugin.png) The Circle CI plugin is available on `/circleci`. -![Circle CI Plugin UI](./architecture-overview/circle-ci.png) +![Circle CI Plugin UI](../assets/architecture-overview/circle-ci.png) -# Plugins and plugin backends +## Plugins and plugin backends Each plugin is a client side application which mounts itself on the UI. Plugins are written in TypeScript or JavaScript. They each live in their own directory @@ -42,7 +48,7 @@ in `backstage/plugins`. For example, the source code for the lighthouse plugin is available at [backstage/plugins/lighthouse](https://github.com/spotify/backstage/tree/master/plugins/lighthouse). -## Installing plugins +### Installing plugins Plugins are typically loaded by the UI in your Backstage applications `plugins.ts` file. For example, @@ -74,7 +80,7 @@ export default builder.build() as ApiHolder; As of this moment, there is no config based install procedure for plugins. Some code changes are required. -## Plugin architecture +### Plugin architecture Architecturally, plugins can take three forms: @@ -82,21 +88,21 @@ Architecturally, plugins can take three forms: 2. Service backed 3. Third-party backed -### Standalone plugins +#### Standalone plugins Standalone plugins run entirely in the browser. [The tech radar plugin](https://backstage-demo.roadie.io/tech-radar), for example, simply renders hard-coded information. It doesn't make any API requests to other services. -![tech radar plugin ui](./architecture-overview/tech-radar-plugin.png) +![tech radar plugin ui](../assets/architecture-overview/tech-radar-plugin.png) The architecture of the Tech Radar installed into a Backstage app is very simple. -![ui and tech radar plugin connected together](./architecture-overview/tech-radar-plugin-architecture.png) +![ui and tech radar plugin connected together](../assets/architecture-overview/tech-radar-plugin-architecture.png) -### Service backed plugins +#### Service backed plugins Service backed plugins make API requests to a service which is within the purview of the organisation running Backstage. @@ -109,7 +115,7 @@ results in a PostgreSQL database. Its architecture looks like this: -![lighthouse plugin backed to microservice and database](./architecture-overview/lighthouse-plugin-architecture.png) +![lighthouse plugin backed to microservice and database](../assets/architecture-overview/lighthouse-plugin-architecture.png) The service catalog in Backstage is another example of a service backed plugin. It retrieves a list of services, or "entities", from the Backstage Backend @@ -131,9 +137,9 @@ Cross Origin Resource Sharing policies which prevent a browser page served at [https://example.com](https://example.com) from serving resources hosted at https://circleci.com. -![CircleCi plugin talking to proxy talking to SaaS Circle CI](./architecture-overview/circle-ci-plugin-architecture.png) +![CircleCi plugin talking to proxy talking to SaaS Circle CI](../assets/architecture-overview/circle-ci-plugin-architecture.png) -# Databases +## Databases As we have seen, both the lighthouse-audit-service and catalog-backend require a database to work with. @@ -150,7 +156,7 @@ GitHub issues. [Update migrations to support postgres by dariddler · Pull Request #1527 · spotify/backstage](https://github.com/spotify/backstage/pull/1527#discussion_r450374145) -# Containerization +## Containerization The example Backstage architecture shown above would Dockerize into three separate docker images. @@ -159,28 +165,26 @@ separate docker images. 2. The backend container 3. The lighthouse audit service container -![Boxes around the architecture to indicate how it is containerised](./architecture-overview/containerised.png) +![Boxes around the architecture to indicate how it is containerised](../assets/architecture-overview/containerised.png) The frontend container can be built with a provided command. ```bash yarn install yarn tsc -yarn build -yarn run docker-build +yarn run docker-build:app ``` Running this will simply generate a Docker container containing the contents of -the UIs `dist` directory. The resulting container will be about 50MB in size. +the UIs `dist` directory. -The backend container can be built by running the following command in the -`packages/backend` directory. +The backend container can be built by running the following command: ```bash -yarn run build-image +yarn run docker-build ``` -This will create a ~500MB container called `example-backend`. +This will create a container called `example-backend`. The lighthouse-audit-service container is already publicly available in Docker Hub and can be downloaded and ran with diff --git a/docs/overview/architecture-terminology.md b/docs/overview/architecture-terminology.md index 6a9d773142..092ca7fb01 100644 --- a/docs/overview/architecture-terminology.md +++ b/docs/overview/architecture-terminology.md @@ -1,4 +1,8 @@ -# Architecture and Terminology +--- +id: architecture-terminology +title: Architecture terminology +description: Documentation on Architecture terminology +--- Backstage is constructed out of three parts. We separate Backstage in this way because we see three groups of contributors that work with Backstage in three diff --git a/docs/overview/background.md b/docs/overview/background.md new file mode 100644 index 0000000000..486260bac3 --- /dev/null +++ b/docs/overview/background.md @@ -0,0 +1,31 @@ +--- +id: background +title: The Spotify Story +description: Documentation on Background and Story behind making of Backstage +--- + +Backstage was born out of necessity at Spotify. We found that as we grew, our +infrastructure was becoming more fragmented, our engineers less productive. + +Instead of building and testing code, teams were spending more time looking for +the right information just to get started. “Where’s the API for that service +we’re all supposed to be using?” “What version of that framework is everyone +on?” “This service isn’t responding, who owns it?” “I can’t find documentation +for anything!” + +Context switching and cognitive overload were dragging engineers down, day by +day. We needed to make it easier for our engineers to do their work without +having to become an expert in every aspect of infrastructure tooling. + +Our idea was to centralize and simplify end-to-end software development with an +abstraction layer that sits on top of all of our infrastructure and developer +tooling. That’s Backstage. + +It’s a developer portal powered by a centralized service catalog — with a plugin +architecture that makes it endlessly extensible and customizable. + +Manage all your services, software, tooling, and testing in Backstage. Start +building a new microservice using an automated template in Backstage. Create, +maintain, and find the documentation for all that software in Backstage. + +One place for everything. Accessible to everyone. diff --git a/docs/overview/logos.md b/docs/overview/logos.md new file mode 100644 index 0000000000..2a684420bc --- /dev/null +++ b/docs/overview/logos.md @@ -0,0 +1,42 @@ +--- +id: logos +title: Logos +sidebar_label: Logo assets +description: Guidelines for how to use the Backstage logos and icons +--- + +Guidelines for how to use the Backstage logo and icon can be found +[here](/logo_assets/Backstage_Identity_Assets_Overview.pdf). The assets below +are all in `.svg` format. Other formats are available in the +[repository](https://github.com/spotify/backstage/tree/master/microsite/static/logo_assets). + +## Backstage logo + + + + + + + + + + + + + +## Backstage icon + +
+ + + + + + + + + + + + +
diff --git a/docs/overview/roadmap.md b/docs/overview/roadmap.md index 36ff658596..d6c1d2d641 100644 --- a/docs/overview/roadmap.md +++ b/docs/overview/roadmap.md @@ -1,36 +1,135 @@ -# Project roadmap +--- +id: roadmap +title: Project roadmap +description: Roadmap of Backstage Project +--- -We created Backstage about 4 years ago. While our internal version of Backstage -has had the benefit of time to mature and evolve, the first iteration of our -open source version is still nascent. We are envisioning three phases of the -project and we have already begun work on various aspects of these phases: +## Current status + +> Backstage is currently under rapid development. This means that you can expect +> APIs and features to evolve. It is also recommended that teams who adopt +> Backstage today upgrade their installation as new +> [releases](https://github.com/spotify/backstage/releases) become available, as +> Backwards compatibility is not yet guaranteed. + +## Phases + +We have divided the project into three high-level _phases_: - 🐣 **Phase 1:** Extensible frontend platform (Done ✅) - You will be able to easily create a single consistent UI layer for your internal infrastructure and tools. A set of reusable - [UX patterns and components](http://storybook.backstage.io) help ensure a + [UX patterns and components](https://backstage.io/storybook) help ensure a consistent experience between tools. - 🐢 **Phase 2:** Service Catalog ([alpha released](https://backstage.io/blog/2020/06/22/backstage-service-catalog-alpha)) - With a single catalog, Backstage makes it easy for a team to manage ten services — and makes it possible for your company to manage thousands of them. - Developers can get a uniform overview of all their software and related - resources, regardless of how and where they are running, as well as an easy - way to onboard and manage those resources. - 🐇 **Phase 3:** Ecosystem (later) - Everyone's infrastructure stack is different. By fostering a vibrant community of contributors we hope to provide an ecosystem of Open Source plugins/integrations that allows you to pick the tools that match your stack. -Check out our [Milestones](https://github.com/spotify/backstage/milestones) and -open [RFCs](https://github.com/spotify/backstage/labels/rfc) how they relate to -the three Phases outlined above. +## Detailed roadmap -Our vision for Backstage is for it to become the trusted standard toolbox (read: -UX layer) for the open source infrastructure landscape. Think of it like -Kubernetes for developer experience. We realize this is an ambitious goal. We -can’t do it alone. If this sounds interesting or you'd like to help us shape our -product vision, we'd love to talk. You can email me directly: +If you have questions about the roadmap or want to provide feedback, we would +love to hear from you! Please create an +[Issue](https://github.com/spotify/backstage/issues/new/choose), ping us on +[Discord](https://discord.gg/EBHEGzX) or reach out directly at [alund@spotify.com](mailto:alund@spotify.com). + +Want to help out? Awesome ❤️ Head over to +[CONTRIBUTING](https://github.com/spotify/backstage/blob/master/CONTRIBUTING.md) +guidelines to get started. + +### Ongoing work 🚧 + +- **[Plugins for managing micro services end-2-end](https://github.com/spotify/backstage/milestone/14)** - + Out of the box Backstage will ship with a set of plugins (Overview, CI, API + and Docs) that will demonstrate how a user can manage a micro service and + follow a change all the way out in production. Completing this work will make + it much easier to see how a plugin can be built that integrates with the + Backstage Service Catalog. + +- **Backstage Design System** - By providing design guidelines for common plugin + layouts together, rich set of reusable UI components + ([Storybook](https://backstage.io/storybook)) and Figma design resources. The + Design System will make it easy to design and build plugins that are + consistent across the platform -- supporting both developers and designers. + +- **[TechDocs v1](https://github.com/spotify/backstage/milestone/16)** - Our + docs-like-code feature TechDocs working end to end. + +- **[Initial GraphQL API](https://github.com/spotify/backstage/milestone/13)** - + A GraphQL API will open up the rich metadata provided by Backstage in a single + query. Plugins can easily query this API as well as extend the model where + needed. + +- **Production deployments** - Provide instructions and default configurations + (e.g. through Helm charts) for easy deployments of Backstage and its + subsystems on Kubernetes. + +- **Cloud Cost Insights plugin (from Spotify)** - Spotify teams are fully + responsible for their own software, including the cost of the cloud resources + they use. By making our internal cost insights plugin available as open source + you will also be able to treat cost as an engineering problem, and make it + easy for your engineers to see their spend and where there's opportunity to + reduce waste. + +- Further improvements to platform documentation + +### Plugins + +Building and maintaining [plugins](https://backstage.io/plugins) is the work of +the entire Backstage community. + +A list of plugins that are in development is +[available here](https://github.com/spotify/backstage/issues?q=is%3Aissue+is%3Aopen+label%3Aplugin+sort%3Areactions-%2B1-desc). +We strongly recommend to upvote 👍 plugins you are interested in. This helps us +and the community prioritize what plugins to build. + +Are you missing a plugin for your favorite tool? Please +[suggest a new one](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME). +Chances are that someone will jump in and help build it. + +### Future work 🔮 + +- **[Backstage platform is stable](https://github.com/spotify/backstage/milestone/19)** - + The platform APIs and features are stable and can be depended on for + production use. After this plugins will require little to no maintenance. + +- **Deploy a product demo at `demo.backstage.io`** - Deploy a typical Backstage + deployment available publicly so that people can click around and get a feel + for the product without having to install anything. + +- **[Global search](https://github.com/spotify/backstage/issues/1499)** - Extend + the basic search available in the Backstage Service Catalog with a global + search experience. Long term this search solution should be extensible, making + it possible for you add custom search results. + +- **[[TechDocs V.2] Stabilization release](https://github.com/spotify/backstage/milestone/17)** - + Platform stability and compatibility improvements. + +- **Additional auth providers** - Backstage should work for most (all!) auth + solutions. Since Backstage can be used by companies regardless of what cloud + (or on prem) you are using we are especially keen to get auth support for + [AWS](https://github.com/spotify/backstage/issues/290), + [Azure](https://github.com/spotify/backstage/issues/348) and others. + +### Completed milestones ✅ + +- [Plugin marketplace](https://backstage.io/plugins) +- [Improved and move documentation to backstage.io](https://backstage.io/docs/overview/what-is-backstage) +- [Backstage Service Catalog (alpha)](https://backstage.io/blog/2020/06/22/backstage-service-catalog-alpha) +- [Backstage Software Templates (alpha)](https://backstage.io/blog/2020/08/05/announcing-backstage-software-templates) +- [Make it possible to add custom auth providers](https://backstage.io/blog/2020/07/01/how-to-enable-authentication-in-backstage-using-passport) +- [TechDocs v0](https://github.com/spotify/backstage/milestone/15) +- CI plugins: CircleCI, Jenkins, GitHub Actions and TravisCI +- [Service API documentation](https://github.com/spotify/backstage/pull/1737) +- Backstage Service Catalog can read from: GitHub, GitLab, + [Bitbucket](https://github.com/spotify/backstage/pull/1938) +- Support auth providers: Google, Okta, GitHub, GitLab, + [auth0](https://github.com/spotify/backstage/pull/1611), + [AWS](https://github.com/spotify/backstage/pull/1990) diff --git a/docs/overview/support.md b/docs/overview/support.md index 7c7390fb69..6e00c17fab 100644 --- a/docs/overview/support.md +++ b/docs/overview/support.md @@ -1,4 +1,8 @@ -# Support and community +--- +id: support +title: Support and community +description: Support and Community Details and Links +--- - [Discord chatroom](https://discord.gg/MUpMjP2) - Get support or discuss the project diff --git a/docs/overview/vision.md b/docs/overview/vision.md new file mode 100644 index 0000000000..df75a4f0a6 --- /dev/null +++ b/docs/overview/vision.md @@ -0,0 +1,23 @@ +--- +id: vision +title: Vision +description: Goal is to provide engineers with the best developer experience in +the world +--- + +Our goal is to provide engineers with the best developer experience in the +world. + +A fantastic developer experience leads to happy, creative and productive +engineers. Our belief is that engineers should not have to be experts in various +infrastructure tools to be productive. Infrastructure should be abstracted away +so that you can return to building and scaling, quickly and safely. + +![](https://backstage.io/animations/backstage-logos-hero-8.gif) + +We are working on making Backstage the trusted standard toolbox (read: UX layer) +for the open source infrastructure landscape. Think of it like Kubernetes for +developer experience. We realize this is an ambitious goal. We can’t do it +alone. If this sounds interesting or you'd like to help us shape our product +vision, we'd love to talk. You can email me directly: +[alund@spotify.com](mailto:alund@spotify.com). diff --git a/docs/overview/what-is-backstage.md b/docs/overview/what-is-backstage.md index eb8ce1f906..0d16943c9b 100644 --- a/docs/overview/what-is-backstage.md +++ b/docs/overview/what-is-backstage.md @@ -1,44 +1,55 @@ -# [Backstage](https://backstage.io) +--- +id: what-is-backstage +title: What is Backstage? +description: Backsatge is an open platform for building developer portals. +Powered by a centralized service catalog, Backstage restores order to your microservices and infrastructure +--- -![headline](../headline.png) - -## What is Backstage? +![service-catalog](https://backstage.io/blog/assets/6/header.png) [Backstage](https://backstage.io/) is an open platform for building developer -portals. It’s based on the developer portal we’ve been using internally at -Spotify for over four years. Backstage can be as simple as a services catalog or -as powerful as the UX layer for your entire tech infrastructure. +portals. Powered by a centralized service catalog, Backstage restores order to +your microservices and infrastructure. So your product teams can ship +high-quality code quickly — without compromising autonomy. -For more information go to [backstage.io](https://backstage.io) or join our -[Discord chatroom](https://discord.gg/EBHEGzX). +Backstage unifies all your infrastructure tooling, services, and documentation +to create a streamlined development environment from end to end. -### Features +Out of the box, Backstage includes: -- Create and manage all of your organization’s software and microservices in one - place. -- Services catalog keeps track of all software and its ownership. -- Visualizations provide information about your backend services and tooling, - and help you monitor them. -- A unified method for managing microservices offers both visibility and - control. -- Preset templates allow engineers to quickly create microservices in a - standardized way - ([coming soon](https://github.com/spotify/backstage/milestone/11)). -- Centralized, full-featured technical documentation with integrated tooling - that makes it easy for developers to set up, publish, and maintain alongside - their code ([coming soon](https://github.com/spotify/backstage/milestone/15)). +- [Backstage Service Catalog](../features/software-catalog/index.md) for + managing all your software (microservices, libraries, data pipelines, + websites, ML models, etc.) + +- [Backstage Software Templates](../features/software-templates/index.md) for + quickly spinning up new projects and standardizing your tooling with your + organization’s best practices + +- [Backstage TechDocs](../features/techdocs/README.md) for making it easy to + create, maintain, find, and use technical documentation, using a "docs like + code" approach + +- Plus, a growing ecosystem of + [open source plugins](https://github.com/spotify/backstage/tree/master/plugins) + that further expand Backstage’s customizability and functionality ### Benefits - For _engineering managers_, it allows you to maintain standards and best practices across the organization, and can help you manage your whole tech ecosystem, from migrations to test certification. + - For _end users_ (developers), it makes it fast and simple to build software components in a standardized way, and it provides a central place to manage all projects and documentation. + - For _platform engineers_, it enables extensibility and scalability by letting you easily integrate new tools and services (via plugins), as well as extending the functionality of existing ones. + - For _everyone_, it’s a single, consistent experience that ties all your infrastructure tooling, resources, standards, owners, contributors, and administrators together in one place. + +If you have questions or want support, please join our +[Discord chatroom](https://discord.gg/EBHEGzX). diff --git a/docs/plugins/add-to-marketplace.md b/docs/plugins/add-to-marketplace.md new file mode 100644 index 0000000000..59cca07bff --- /dev/null +++ b/docs/plugins/add-to-marketplace.md @@ -0,0 +1,26 @@ +--- +id: add-to-marketplace +title: Add to Marketplace +description: Documentation on Adding Plugin to Marketplace +--- + +## Adding a Plugin to the Marketplace + +To add a new plugin to the [plugin marketplace](https://backstage.io/plugins) +create a file in +[`microsite/data/plugins`](https://github.com/spotify/backstage/tree/master/microsite/data/plugins) +with your plugin's information. Example: + +```yaml +--- +title: Your Plugin +author: Your Name +authorUrl: # A link to information about the author E.g. Company url, github user profile, etc +category: Monitoring # A single category e.g. CI, Machine Learning, Services, Monitoring +description: A brief description of the plugin. # Max 170 characters +documentation: # A link to your documentation E.g. Your github README +iconUrl: # Used as the src attribute for your logo. +# You can provide an external url or add your logo under static/img and provide a path +# relative to static/ e.g. img/my-logo.png +npmPackageName: # Your npm package name E.g. '@backstage/plugin-' quotes are required +``` diff --git a/docs/plugins/backend-plugin.md b/docs/plugins/backend-plugin.md index e69de29bb2..516ba4b111 100644 --- a/docs/plugins/backend-plugin.md +++ b/docs/plugins/backend-plugin.md @@ -0,0 +1,7 @@ +--- +id: backend-plugin +title: Backend plugin +description: Documentation on Backend plugin +--- + +## TODO diff --git a/docs/plugins/call-existing-api.md b/docs/plugins/call-existing-api.md index e69de29bb2..14ea4c1f5a 100644 --- a/docs/plugins/call-existing-api.md +++ b/docs/plugins/call-existing-api.md @@ -0,0 +1,174 @@ +--- +id: call-existing-api +title: Call Existing API +description: Describes the various options that Backstage frontend plugins have, +in communicating with service APIs that already exist +--- + +This article describes the various options that Backstage frontend plugins have, +in communicating with service APIs that already exist. Each section below +describes a possible choice, and the circumstances under which it fits. + +In these examples, we will be ultimately requesting data from the fictional +FrobsCo API. + +## Issuing Requests Directly + +The most basic choice available is to issue requests directly from the plugin +frontend code to the FrobsCo API, using for example `fetch` or a support library +such as `axios`. + +Example: + +```ts +// Inside your component +fetch('https://api.frobsco.com/v1/list') + .then(response => response.json()) + .then(payload => setFrobs(payload as Frob[])); +``` + +Internally at Spotify, this has not been a very common choice. Third party APIs +are sometimes accessed like this. Just a handful of internal APIs also went +through the trouble of exposing themselves in a way that is useful directly from +a browser, but even then, often not from the public internet but only supporting +users that are already on the company VPN. + +This can be used when: + +- The API already does/exposes exactly what you need. +- The request/response patterns of the API match real world usage needs in + Backstage frontend plugins. For example, if the end use case is to show a + small summary in Backstage, but the only available API endpoint gives a 30 + megabyte blob with large amounts of redundant information, it would hurt the + end user experience. Particularly on mobile. The same goes for cases where you + want to show many individual pieces of information: if a common use case is to + show large tables where one API request per cell is necessary, the browser + will quickly become swamped and you may want to consider performing + aggregation elsewhere instead. +- The API can maintain interactive request/response times at your required peak + request rates. The end user experience will be degraded if they spend a lot of + time waiting for the data to arrive. +- The API endpoint is highly available. The browser does not have builtin + facilities for load balancing, service discovery, retries, health checks, + circuit breaking and similar. If the endpoint is occasionally down even for + short periods of time (e.g. during deploys), end users will quickly notice. +- The API is exposed over HTTPS (not just HTTP), and properly handles + [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS). These are + requirements that the user's browser will impose for security reasons, and the + requests will be rejected otherwise. +- The API endpoint is easily reachable, in terms of network conditions, by end + users. This may be particularly relevant if your end users are outside of your + perimeter. +- The requests do not require secrets to be passed. This limitation does not + apply to OAuth tokens, which the frontend can negotiate and make proper use + of. + +## Using The Backstage Proxy + +Backstage has an optional proxy plugin for the backend, that can be used to +easily add proxy routes to downstream APIs. + +Example: + +```yaml +# In app-config.yaml +proxy: + '/frobs': http://api.frobsco.com/v1 +``` + +```ts +// Inside your component +const backendUrl = config.getString('backend.baseUrl'); +fetch(`${backendUrl}/proxy/frobs/list`) + .then(response => response.json()) + .then(payload => setFrobs(payload as Frob[])); +``` + +The proxy is powered by the `http-proxy-middleware` package. See +[Proxying](proxying.md) for a full description of its configuration options. + +Internally at Spotify, the proxy option has been the overwhelmingly most popular +choice for plugin makers. Since we have DNS based service discovery in place and +a microservices framework that made it trivial to expose plain HTTP, it has been +a matter of just adding a few lines of Backstage config to get the benefit of +being easily and robustly reachable from users' web browsers as well. + +This may be used instead of direct requests, when: + +- You need to perform HTTPS termination and/or CORS handling, because the API + itself is not supplying those. +- You need to inject a simple static secret into the requests, e.g. an + Authorization header that gets added to the request headers. +- You want to make use of other proxy facilities, such as retries, failover, + health checks, routing, request logging, rewrites, etc. +- You already have the Backstage backend itself exposed through your perimeter + and find it practical to have only one entry point to deal with, governing + ingress with just the Backstage config. + +## Creating a Backstage Backend Plugin + +Much like the Backstage frontend, the Backstage backend also has a plugin +system. The above mentioned proxy is actually one such plugin. If you were in +need of a more involved integration than just direct access to the FrobsCo API, +or if you needed to hold state, you may want to make such a plugin. + +Example: + +```ts +// Inside your component +const backendUrl = config.getString('backend.baseUrl'); +fetch(`${backendUrl}/frobs-aggregator/summary`) + .then(response => response.json()) + .then(payload => setSummary(payload as FrobSummary)); +``` + +```ts +// Inside a new frobs-aggregator backend plugin +router.use('/summary', async (req, res) => { + const agg = await Promise.all([ + fetch('https://api.frobsco.com/v1/list'), + fetch('http://flerps.partnercompany.com:8080/flerp-batch'), + database.currentThunk(), + ]).then(async ([frobs, flerps, thunk]) => { + return computeAggregate(await frobs.json(), await flerps.json(), thunk); + }); + res.status(200).send(agg); +}); +``` + +For a more detailed example, see +[the lighthouse plugin](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) +that stores some state in a database and adds new capabilities to the underlying +API. + +Internally at Spotify, this has been a fairly popular choice for different +reasons. Commonly, the backend has been used as a caching and data massaging +layer for slow APIs or APIs whose request/response shapes or speeds were not +acceptable for direct use by frontends. For example, this has made it possible +to issue efficient batch queries from the frontend, e.g. in big lists or tables +that want to resolve a lot of sparse data from the larger list that an +underlying service supplies. + +This may be used instead of the above, when: + +- You need to perform complex model conversion, or protocol translation beyond + what the proxy handles. +- You want to perform aggregations or summaries on the backend instead of on the + frontend. +- You want to enable batching or caching of slower or more unreliable APIs. +- You need to maintain state for your plugin, perhaps using the builtin database + support in the backend. +- You need to inject secrets or in other ways negotiate with other parts of the + API or other services in order to perform your work. +- You want to enforce end user authentication / authorization for operations on + behalf of the API, have session handling, or similar. + +There is a balance to strike regarding when to make an entirely separate backend +for a purpose, and when to make a Backstage backend plugin that adapts something +that already exists. General advice is not easy to give, but contact us on +Discord if you have any questions, and we may be able to offer guidance. + +## Extending the GraphQL Model + +The extensible GraphQL backend layer is not built yet. This section will be +expanded when that happens. Stay tuned! diff --git a/docs/plugins/create-a-plugin.md b/docs/plugins/create-a-plugin.md index fa91e38b06..bcb86024fd 100644 --- a/docs/plugins/create-a-plugin.md +++ b/docs/plugins/create-a-plugin.md @@ -1,4 +1,8 @@ -# Create a Backstage Plugin +--- +id: create-a-plugin +title: Create a Backstage Plugin +description: Documentation on How to Create a Backstage Plugin +--- A Backstage Plugin adds functionality to Backstage. @@ -12,20 +16,16 @@ dependencies, then run the following on your command line (invoking the yarn create-plugin ``` -

- create plugin -

+![](../assets/getting-started/create-plugin_output.png) This will create a new Backstage Plugin based on the ID that was provided. It will be built and added to the Backstage App automatically. -_If `yarn start` is already running you should be able to see the default page -for your new plugin directly by navigating to -`http://localhost:3000/my-plugin`._ +> If `yarn start` is already running you should be able to see the default page +> for your new plugin directly by navigating to +> `http://localhost:3000/my-plugin`. -

- my plugin -

+![](../assets/my-plugin_screenshot.png) You can also serve the plugin in isolation by running `yarn start` in the plugin directory. Or by using the yarn workspace command, for example: @@ -37,7 +37,3 @@ yarn workspace @backstage/plugin-welcome start # Also supports --check This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. It is only meant for local development, and the setup for it can be found inside the plugin's `dev/` directory. - -[Next Step - Structure of a plugin](structure-of-a-plugin.md) - -[Back to Getting Started](../README.md) diff --git a/docs/plugins/existing-plugins.md b/docs/plugins/existing-plugins.md index bd6fd19691..c1fbda379a 100644 --- a/docs/plugins/existing-plugins.md +++ b/docs/plugins/existing-plugins.md @@ -1,4 +1,8 @@ -# Existing plugins +--- +id: existing-plugins +title: Existing plugins +description: Lists of existing open source plugins +--- ## Open source plugins diff --git a/docs/plugins/index.md b/docs/plugins/index.md index 5021ac9896..8a61aeee80 100644 --- a/docs/plugins/index.md +++ b/docs/plugins/index.md @@ -1,4 +1,8 @@ -# Plugins +--- +id: index +title: Intro to plugins +description: Documentation on Introduction to Plugins +--- Backstage is a single-page application composed of a set of plugins. @@ -8,7 +12,7 @@ development tool as a plugin in Backstage. By following strong [design guidelines](../dls/design.md) we ensure the the overall user experience stays consistent between plugins. -![plugin](my-plugin_screenshot.png) +![plugin](../assets/my-plugin_screenshot.png) ## Creating a plugin @@ -18,8 +22,15 @@ To create a plugin, follow the steps outlined [here](create-a-plugin.md). If you start developing a plugin that you aim to release as open source, we suggest that you create a new -[new Issue](https://github.com/spotify/backstage/issues/new?template=plugin_template.md). +[new Issue](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME). This helps the community know what plugins are in development. You can also use this process if you have an idea for a good plugin but you hope that someone else will pick up the work. + +## Integrate into the Service Catalog + +If your plugin isn't supposed to live as a standalone page, but rather needs to +be presented as a part of a Service Catalog (e.g. a separate tab or a card on an +"Overview" tab), then check out +[the instruction](integrating-plugin-into-service-catalog.md). on how to do it. diff --git a/docs/plugins/integrating-plugin-into-service-catalog.md b/docs/plugins/integrating-plugin-into-service-catalog.md new file mode 100644 index 0000000000..51d2331140 --- /dev/null +++ b/docs/plugins/integrating-plugin-into-service-catalog.md @@ -0,0 +1,125 @@ +--- +id: integrating-plugin-into-service-catalog +title: Integrate into the Service Catalog +description: Documentation on How to integrate plugin into service catalog +--- + +> This is an advanced use case and currently is an experimental feature. Expect +> API to change over time + +## Steps + +1. [Create a plugin](#create-a-plugin) +1. [Export a router with relative routes](#export-a-router) +1. [Import and use router in the APP](#import-and-use-router-in-the-app) + +### Create a plugin + +Follow the [same process](create-a-plugin.md) as for standalone plugin. You +should have a separate package in a folder, which represents your plugin. + +Example: + +``` +$ yarn create-plugin +> ? Enter an ID for the plugin [required] my-plugin +> ? Enter the owner(s) of the plugin. If specified, this will be added to CODEOWNERS for the plugin path. [optional] + +Creating the plugin... +``` + +### Export a router + +Now in the plugin you have a `Router.tsx` file in the `src` folder. By default +it contains only one example route. Create a routing structure needed for your +plugin, keeping in mind that the whole set of routes defined here are going to +be mounted under some different route in the App. + +Example: + +`my-plugin` consists of 2 different views - `/me` and `/about`. I envision +people integrating it into plugin catalog as a tab named "MyPlugin". Then, my +`Routes.tsx` for the plugin is going to look like: + +```tsx + + } /> + } /> + +``` + +(where MePage and AboutPage are 2 components defined in your plugin and imported +accordingly inside `Router.tsx`) + +> Pay attention, if your `MePage` references the `AboutPage` it needs to do it +> through link to `about`, not `/about`. This allows react-router v6 to enable +> its relative routing mechanism. Read more - +> https://reacttraining.com/blog/react-router-v6-pre/#relative-route-path-and-link-to + +### Import and use router in the APP + +In the `app/src/components/catalog/EntityPage.tsx` (app === your folder, +containing backstage app) import your created Router: + +```tsx +import { Router as MyPluginRouter } from '@backstage/plugin-my-plugin; +``` + +Now, you need to mount `MyPluginRouter` onto some route, for example if you had: + +```tsx +const DefaultEntityPage = ({ entity }: { entity: Entity }) => ( + + } + /> + +); +``` + +after you add your code it becomes: + +```tsx +const DefaultEntityPage = ({ entity }: { entity: Entity }) => ( + + } + /> + } + /> + +); +``` + +All of magic happens thanks to the `EntityPageLayout` component, which comes as +an export from `@backstage/plugin-catalog` package. + +```tsx +type EntityPageLayoutContentProps = { + /** + * Going to be transformed into react-router v6 + * path under the hood. Read more at https://reacttraining.com/blog/react-router-v6-pre + */ + path: string; + /** + * Gets transformed into the title for the tab + */ + title: string; + /** + * Element that is rendered when the location + * matches the path provided + */ + element: JSX.Element; +}; +``` + +> You can either pass the entity from App to the plugin's router as a prop or +> use `useEntity` hook from `@backstage/plugin-catalog` directly inside your +> plugin. diff --git a/docs/plugins/plugin-development.md b/docs/plugins/plugin-development.md index 41918b5de2..c5d9763f7d 100644 --- a/docs/plugins/plugin-development.md +++ b/docs/plugins/plugin-development.md @@ -1,4 +1,8 @@ -# Plugin Development in Backstage +--- +id: plugin-development +title: Plugin Development +description: Documentation on Plugin Development +--- Backstage plugins provide features to a Backstage App. @@ -7,28 +11,12 @@ type of content. Plugins all use a common set of platform APIs and reusable UI components. Plugins can fetch data from external sources using the regular browser APIs or by depending on external modules to do the work. - - ## Developing guidelines - Consider writing plugins in `TypeScript`. - Plan the directory structure of your plugin so that it becomes easy to manage. -- Prefer using the Backstage components, otherwise go with - [Material-UI](https://material-ui.com/). +- Prefer using the [Backstage components](https://backstage.io/storybook), + otherwise go with [Material-UI](https://material-ui.com/). - Check out the shared Backstage APIs before building a new one. ## Plugin concepts / API diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index e69de29bb2..2f809ffd10 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -0,0 +1,79 @@ +--- +id: proxying +title: Proxying +description: Documentation on Proxying +--- + +## Overview + +The Backstage backend comes packaged with a basic HTTP proxy, that can aid in +reaching backend service APIs from frontend plugin code. See +[Call Existing API](call-existing-api.md) for a description of when the proxy +can be the best choice for communicating with an API. + +## Getting Started + +The plugin is already added to a default Backstage project. + +In `packages/backend/src/index.ts`: + +```ts +const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + +const service = createServiceBuilder(module) + .loadConfig(configReader) + /** ... other routers ... */ + .addRouter('/proxy', await proxy(proxyEnv, '/proxy')); +``` + +## Configuration + +Configuration for the proxy plugin lives under a `proxy` root key of your +`app-config.yaml` file. + +Example: + +```yaml +# in app-config.yaml +proxy: + '/simple-example': http://simple.example.com:8080 + '/larger-example/v1': + target: http://larger.example.com:8080/svc.v1 + headers: + Authorization: + $secret: + env: EXAMPLE_AUTH_HEADER +``` + +Each key under the proxy configuration entry is a route to match, below the +prefix that the proxy plugin is mounted on. It must start with a slash. For +example, if the backend mounts the proxy plugin as `/proxy`, the above +configuration will lead to the proxy acting on backend requests to +`/proxy/simple-example/...` and `/proxy/larger-example/v1/...`. + +The value inside each route is either a simple URL string, or an object on the +format accepted by +[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). It +is also possible to limit the forwarded HTTP methods with the configuration +`allowedMethods`, for example `allowedMethods: ['GET']` to enforce read-only +access. + +If the value is a string, it is assumed to correspond to: + +```yaml +target: +changeOrigin: true +pathRewrite: + '^/': '/' +``` + +When the target is an object, it is given verbatim to `http-proxy-middleware` +except with the following caveats for convenience: + +- If `changeOrigin` is not specified, it is set to `true`. This is the most + commonly useful value. +- If `pathRewrite` is not specified, it is set to a single rewrite that removes + the entire prefix and route. In the above example, a rewrite of + `'^/proxy/larger-example/v1/': '/'` is added. That means that a request to + `/proxy/larger-example/v1/some/path` will be translated to a request to + `http://larger.example.com:8080/svc.v1/some/path`. diff --git a/docs/plugins/publish-private.md b/docs/plugins/publish-private.md index e69de29bb2..d04449d465 100644 --- a/docs/plugins/publish-private.md +++ b/docs/plugins/publish-private.md @@ -0,0 +1,7 @@ +--- +id: publish-private +title: Publish private +description: Documentation on How to Publish private +--- + +## TODO diff --git a/docs/plugins/publishing.md b/docs/plugins/publishing.md index 76997c30d2..0ed85e8cdc 100644 --- a/docs/plugins/publishing.md +++ b/docs/plugins/publishing.md @@ -1,4 +1,8 @@ -# Publishing +--- +id: publishing +title: Publishing +description: Documentation on Publishing NPM packages +--- ## NPM @@ -36,4 +40,18 @@ $ git push origin -u new-release And then create a PR. Once the PR is approved and merged into master, the master build will publish new versions of all bumped packages. +### Include new changes in existing release PR + +If you want to include some last minute changes to an existing release PR, +follow these instructions: + +```sh +$ git checkout master +$ git pull +$ git checkout new-release +$ git reset --hard master +$ yarn release +$ git push --force +``` + [Back to Docs](../README.md) diff --git a/docs/plugins/structure-of-a-plugin.md b/docs/plugins/structure-of-a-plugin.md index ceacf578fe..f061d2f4a8 100644 --- a/docs/plugins/structure-of-a-plugin.md +++ b/docs/plugins/structure-of-a-plugin.md @@ -1,4 +1,8 @@ -# Structure of a Plugin +--- +id: structure-of-a-plugin +title: Structure of a Plugin +description: Details about structure of a plugin +--- Nice, you have a new plugin! We'll soon see how we can develop it into doing great things. But first off, let's look at what we get out of the box. diff --git a/docs/plugins/testing.md b/docs/plugins/testing.md index 701db3eff4..696f8b1368 100644 --- a/docs/plugins/testing.md +++ b/docs/plugins/testing.md @@ -1,4 +1,8 @@ -# Testing with Jest +--- +id: testing +title: Testing with Jest +description: Documentation on How to do unit testing with Jest +--- Backstage uses [Jest](https://facebook.github.io/jest/) for all our unit testing needs. diff --git a/docs/reference/createPlugin-feature-flags.md b/docs/reference/createPlugin-feature-flags.md index 0f1debd515..ebb8a6b503 100644 --- a/docs/reference/createPlugin-feature-flags.md +++ b/docs/reference/createPlugin-feature-flags.md @@ -1,4 +1,8 @@ -# createPlugin - feature flags +--- +id: createPlugin-feature-flags +title: createPlugin - feature flags +description: Documentation on createPlugin - feature flags +--- The `featureFlags` object passed to the `register` function makes it possible for plugins to register Feature Flags in Backstage for users to opt into. You diff --git a/docs/reference/createPlugin-router.md b/docs/reference/createPlugin-router.md index 831b27f388..89ee44e558 100644 --- a/docs/reference/createPlugin-router.md +++ b/docs/reference/createPlugin-router.md @@ -1,4 +1,8 @@ -# createPlugin - router +--- +id: createPlugin-router +title: createPlugin - router +description: Documentation on createPlugin - router +--- The router that is passed to the `register` function makes it possible for plugins to hook into routing of the Backstage app and provide the end users with @@ -34,5 +38,3 @@ const myPluginRouteRef = createRouteRef({ title: 'My Plugin', }); ``` - -[Back to References](../README.md) diff --git a/docs/reference/createPlugin.md b/docs/reference/createPlugin.md index 7ecb6d461a..45e3303124 100644 --- a/docs/reference/createPlugin.md +++ b/docs/reference/createPlugin.md @@ -1,4 +1,8 @@ -# createPlugin +--- +id: createPlugin +title: createPlugin +description: Documentation on createPlugin +--- Taking a plugin config as argument and returns a new plugin. diff --git a/docs/reference/utility-apis/AlertApi.md b/docs/reference/utility-apis/AlertApi.md new file mode 100644 index 0000000000..d000a6d6a2 --- /dev/null +++ b/docs/reference/utility-apis/AlertApi.md @@ -0,0 +1,114 @@ +# AlertApi + +The AlertApi type is defined at +[packages/core-api/src/apis/definitions/AlertApi.ts:29](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AlertApi.ts#L29). + +The following Utility API implements this type: [alertApiRef](./README.md#alert) + +## Members + +### post() + +Post an alert for handling by the application. + +
+post(alert: AlertMessage): void
+
+ +### alert\$() + +Observe alerts posted by other parts of the application. + +
+alert$(): Observable<AlertMessage>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AlertMessage + +
+export type AlertMessage = {
+  message: string;
+  // Severity will default to success since that is what material ui defaults the value to.
+  severity?: 'success' | 'info' | 'warning' | 'error';
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/AlertApi.ts:19](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AlertApi.ts#L19). + +Referenced by: [post](#post), [alert\$](#alert). + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [alert\$](#alert). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/reference/utility-apis/AppThemeApi.md b/docs/reference/utility-apis/AppThemeApi.md new file mode 100644 index 0000000000..e7d5296ceb --- /dev/null +++ b/docs/reference/utility-apis/AppThemeApi.md @@ -0,0 +1,232 @@ +# AppThemeApi + +The AppThemeApi type is defined at +[packages/core-api/src/apis/definitions/AppThemeApi.ts:50](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AppThemeApi.ts#L50). + +The following Utility API implements this type: +[appThemeApiRef](./README.md#apptheme) + +## Members + +### getInstalledThemes() + +Get a list of available themes. + +
+getInstalledThemes(): AppTheme[]
+
+ +### activeThemeId\$() + +Observe the currently selected theme. A value of undefined means no specific +theme has been selected. + +
+activeThemeId$(): Observable<string | undefined>
+
+ +### getActiveThemeId() + +Get the current theme ID. Returns undefined if no specific theme is selected. + +
+getActiveThemeId(): string | undefined
+
+ +### setActiveThemeId() + +Set a specific theme to use in the app, overriding the default theme selection. + +Clear the selection by passing in undefined. + +
+setActiveThemeId(themeId?: string): void
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AppTheme + +Describes a theme provided by the app. + +
+export type AppTheme = {
+  /**
+   * ID used to remember theme selections.
+   */
+  id: string;
+
+  /**
+   * Title of the theme
+   */
+  title: string;
+
+  /**
+   * Theme variant
+   */
+  variant: 'light' | 'dark';
+
+  /**
+   * The specialized MaterialUI theme instance.
+   */
+  theme: BackstageTheme;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/AppThemeApi.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AppThemeApi.ts#L24). + +Referenced by: [getInstalledThemes](#getinstalledthemes). + +### BackstagePalette + +
+export type BackstagePalette = Palette & PaletteAdditions
+
+ +Defined at +[packages/theme/src/types.ts:70](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/theme/src/types.ts#L70). + +Referenced by: [BackstageTheme](#backstagetheme). + +### BackstageTheme + +
+export interface BackstageTheme extends Theme {
+  palette: BackstagePalette;
+}
+
+ +Defined at +[packages/theme/src/types.ts:73](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/theme/src/types.ts#L73). + +Referenced by: [AppTheme](#apptheme). + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [activeThemeId\$](#activethemeid). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### PaletteAdditions + +
+type PaletteAdditions = {
+  status: {
+    ok: string;
+    warning: string;
+    error: string;
+    pending: string;
+    running: string;
+    aborted: string;
+  };
+  border: string;
+  textContrast: string;
+  textVerySubtle: string;
+  textSubtle: string;
+  highlight: string;
+  errorBackground: string;
+  warningBackground: string;
+  infoBackground: string;
+  errorText: string;
+  infoText: string;
+  warningText: string;
+  linkHover: string;
+  link: string;
+  gold: string;
+  navigation: {
+    background: string;
+    indicator: string;
+  };
+  tabbar: {
+    indicator: string;
+  };
+  bursts: {
+    fontColor: string;
+    slackChannelText: string;
+    backgroundColor: {
+      default: string;
+    };
+  };
+  pinSidebarButton: {
+    icon: string;
+    background: string;
+  };
+  banner: {
+    info: string;
+    error: string;
+  };
+}
+
+ +Defined at +[packages/theme/src/types.ts:23](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/theme/src/types.ts#L23). + +Referenced by: [BackstagePalette](#backstagepalette). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/reference/utility-apis/BackstageIdentityApi.md b/docs/reference/utility-apis/BackstageIdentityApi.md new file mode 100644 index 0000000000..87318ecff5 --- /dev/null +++ b/docs/reference/utility-apis/BackstageIdentityApi.md @@ -0,0 +1,92 @@ +# BackstageIdentityApi + +The BackstageIdentityApi type is defined at +[packages/core-api/src/apis/definitions/auth.ts:144](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L144). + +The following Utility APIs implement this type: + +- [auth0AuthApiRef](./README.md#auth0auth) + +- [githubAuthApiRef](./README.md#githubauth) + +- [gitlabAuthApiRef](./README.md#gitlabauth) + +- [googleAuthApiRef](./README.md#googleauth) + +- [microsoftAuthApiRef](./README.md#microsoftauth) + +- [oktaAuthApiRef](./README.md#oktaauth) + +## Members + +### getBackstageIdentity() + +Get the user's identity within Backstage. This should normally not be called +directly, use the @IdentityApi instead. + +If the optional flag is not set, a session is guaranteed to be returned, while +if the optional flag is set, the session may be undefined. See +@AuthRequestOptions for more details. + +
+getBackstageIdentity(
+    options?: AuthRequestOptions,
+  ): Promise<BackstageIdentity | undefined>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AuthRequestOptions + +
+export type AuthRequestOptions = {
+  /**
+   * If this is set to true, the user will not be prompted to log in,
+   * and an empty response will be returned if there is no existing session.
+   *
+   * This can be used to perform a check whether the user is logged in, or if you don't
+   * want to force a user to be logged in, but provide functionality if they already are.
+   *
+   * @default false
+   */
+  optional?: boolean;
+
+  /**
+   * If this is set to true, the request will bypass the regular oauth login modal
+   * and open the login popup directly.
+   *
+   * The method must be called synchronously from a user action for this to work in all browsers.
+   *
+   * @default false
+   */
+  instantPopup?: boolean;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:40](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L40). + +Referenced by: [getBackstageIdentity](#getbackstageidentity). + +### BackstageIdentity + +
+export type BackstageIdentity = {
+  /**
+   * The backstage user ID.
+   */
+  id: string;
+
+  /**
+   * An ID token that can be used to authenticate the user within Backstage.
+   */
+  idToken: string;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:157](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L157). + +Referenced by: [getBackstageIdentity](#getbackstageidentity). diff --git a/docs/reference/utility-apis/Config.md b/docs/reference/utility-apis/Config.md new file mode 100644 index 0000000000..9374fb9962 --- /dev/null +++ b/docs/reference/utility-apis/Config.md @@ -0,0 +1,187 @@ +# Config + +The Config type is defined at +[packages/config/src/types.ts:32](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/config/src/types.ts#L32). + +The following Utility API implements this type: +[configApiRef](./README.md#config) + +## Members + +### has() + +
+has(key: string): boolean
+
+ +### keys() + +
+keys(): string[]
+
+ +### get() + +
+get(key?: string): JsonValue
+
+ +### getOptional() + +
+getOptional(key?: string): JsonValue | undefined
+
+ +### getConfig() + +
+getConfig(key: string): Config
+
+ +### getOptionalConfig() + +
+getOptionalConfig(key: string): Config | undefined
+
+ +### getConfigArray() + +
+getConfigArray(key: string): Config[]
+
+ +### getOptionalConfigArray() + +
+getOptionalConfigArray(key: string): Config[] | undefined
+
+ +### getNumber() + +
+getNumber(key: string): number
+
+ +### getOptionalNumber() + +
+getOptionalNumber(key: string): number | undefined
+
+ +### getBoolean() + +
+getBoolean(key: string): boolean
+
+ +### getOptionalBoolean() + +
+getOptionalBoolean(key: string): boolean | undefined
+
+ +### getString() + +
+getString(key: string): string
+
+ +### getOptionalString() + +
+getOptionalString(key: string): string | undefined
+
+ +### getStringArray() + +
+getStringArray(key: string): string[]
+
+ +### getOptionalStringArray() + +
+getOptionalStringArray(key: string): string[] | undefined
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### Config + +
+export type Config = {
+  has(key: string): boolean;
+
+  keys(): string[];
+
+  get(key?: string): JsonValue;
+  getOptional(key?: string): JsonValue | undefined;
+
+  getConfig(key: string): Config;
+  getOptionalConfig(key: string): Config | undefined;
+
+  getConfigArray(key: string): Config[];
+  getOptionalConfigArray(key: string): Config[] | undefined;
+
+  getNumber(key: string): number;
+  getOptionalNumber(key: string): number | undefined;
+
+  getBoolean(key: string): boolean;
+  getOptionalBoolean(key: string): boolean | undefined;
+
+  getString(key: string): string;
+  getOptionalString(key: string): string | undefined;
+
+  getStringArray(key: string): string[];
+  getOptionalStringArray(key: string): string[] | undefined;
+}
+
+ +Defined at +[packages/config/src/types.ts:32](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/config/src/types.ts#L32). + +Referenced by: [getConfig](#getconfig), [getOptionalConfig](#getoptionalconfig), +[getConfigArray](#getconfigarray), +[getOptionalConfigArray](#getoptionalconfigarray), [Config](#config). + +### JsonArray + +
+export type JsonArray = JsonValue[]
+
+ +Defined at +[packages/config/src/types.ts:18](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/config/src/types.ts#L18). + +Referenced by: [JsonValue](#jsonvalue). + +### JsonObject + +
+export type JsonObject = { [key in string]?: JsonValue }
+
+ +Defined at +[packages/config/src/types.ts:17](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/config/src/types.ts#L17). + +Referenced by: [JsonValue](#jsonvalue). + +### JsonValue + +
+export type JsonValue =
+  | JsonObject
+  | JsonArray
+  | number
+  | string
+  | boolean
+  | null
+
+ +Defined at +[packages/config/src/types.ts:19](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/config/src/types.ts#L19). + +Referenced by: [get](#get), [getOptional](#getoptional), +[JsonObject](#jsonobject), [JsonArray](#jsonarray), [Config](#config). diff --git a/docs/reference/utility-apis/DiscoveryApi.md b/docs/reference/utility-apis/DiscoveryApi.md new file mode 100644 index 0000000000..39902789cd --- /dev/null +++ b/docs/reference/utility-apis/DiscoveryApi.md @@ -0,0 +1,24 @@ +# DiscoveryApi + +The DiscoveryApi type is defined at +[packages/core-api/src/apis/definitions/DiscoveryApi.ts:30](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/DiscoveryApi.ts#L30). + +The following Utility API implements this type: +[discoveryApiRef](./README.md#discovery) + +## Members + +### getBaseUrl() + +Returns the HTTP base backend URL for a given plugin, without a trailing slash. + +This method must always be called just before making a request. as opposed to +fetching the URL when constructing an API client. That is to ensure that more +flexible routing patterns can be supported. + +For example, asking for the URL for `auth` may return something like +`https://backstage.example.com/api/auth` + +
+getBaseUrl(pluginId: string): Promise<string>
+
diff --git a/docs/reference/utility-apis/ErrorApi.md b/docs/reference/utility-apis/ErrorApi.md new file mode 100644 index 0000000000..c05f060ec3 --- /dev/null +++ b/docs/reference/utility-apis/ErrorApi.md @@ -0,0 +1,134 @@ +# ErrorApi + +The ErrorApi type is defined at +[packages/core-api/src/apis/definitions/ErrorApi.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/ErrorApi.ts#L53). + +The following Utility API implements this type: [errorApiRef](./README.md#error) + +## Members + +### post() + +Post an error for handling by the application. + +
+post(error: Error, context?: ErrorContext): void
+
+ +### error\$() + +Observe errors posted by other parts of the application. + +
+error$(): Observable<{ error: Error; context?: ErrorContext }>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### Error + +Mirrors the javascript Error class, for the purpose of providing documentation +and optional fields. + +
+type Error = {
+  name: string;
+  message: string;
+  stack?: string;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/ErrorApi.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/ErrorApi.ts#L24). + +Referenced by: [post](#post), [error\$](#error). + +### ErrorContext + +Provides additional information about an error that was posted to the +application. + +
+export type ErrorContext = {
+  // If set to true, this error should not be displayed to the user. Defaults to false.
+  hidden?: boolean;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/ErrorApi.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/ErrorApi.ts#L33). + +Referenced by: [post](#post), [error\$](#error). + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [error\$](#error). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/reference/utility-apis/FeatureFlagsApi.md b/docs/reference/utility-apis/FeatureFlagsApi.md new file mode 100644 index 0000000000..5efdb176a4 --- /dev/null +++ b/docs/reference/utility-apis/FeatureFlagsApi.md @@ -0,0 +1,33 @@ +# FeatureFlagsApi + +The FeatureFlagsApi type is defined at +[packages/core-api/src/apis/definitions/FeatureFlagsApi.ts:41](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts#L41). + +The following Utility API implements this type: +[featureFlagsApiRef](./README.md#featureflags) + +## Members + +### registeredFeatureFlags + +Store a list of registered feature flags. + +
+registeredFeatureFlags: FeatureFlagsRegistryItem[]
+
+ +### getFlags() + +Get a list of all feature flags from the current user. + +
+getFlags(): UserFlags
+
+ +### getRegisteredFlags() + +Get a list of all registered flags. + +
+getRegisteredFlags(): FeatureFlagsRegistry
+
diff --git a/docs/reference/utility-apis/IdentityApi.md b/docs/reference/utility-apis/IdentityApi.md new file mode 100644 index 0000000000..4e37ad5300 --- /dev/null +++ b/docs/reference/utility-apis/IdentityApi.md @@ -0,0 +1,81 @@ +# IdentityApi + +The IdentityApi type is defined at +[packages/core-api/src/apis/definitions/IdentityApi.ts:22](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/IdentityApi.ts#L22). + +The following Utility API implements this type: +[identityApiRef](./README.md#identity) + +## Members + +### getUserId() + +The ID of the signed in user. This ID is not meant to be presented to the user, +but used as an opaque string to pass on to backends or use in frontend logic. + +TODO: The intention of the user ID is to be able to tie the user to an identity +that is known by the catalog and/or identity backend. It should for example be +possible to fetch all owned components using this ID. + +
+getUserId(): string
+
+ +### getProfile() + +The profile of the signed in user. + +
+getProfile(): ProfileInfo
+
+ +### getIdToken() + +An OpenID Connect ID Token which proves the identity of the signed in user. + +The ID token will be undefined if the signed in user does not have a verified +identity, such as a demo user or mocked user for e2e tests. + +
+getIdToken(): Promise<string | undefined>
+
+ +### logout() + +Log out the current user + +
+logout(): Promise<void>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### ProfileInfo + +Profile information of the user. + +
+export type ProfileInfo = {
+  /**
+   * Email ID.
+   */
+  email?: string;
+
+  /**
+   * Display name that can be presented to the user.
+   */
+  displayName?: string;
+
+  /**
+   * URL to an avatar image of the user.
+   */
+  picture?: string;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:172](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L172). + +Referenced by: [getProfile](#getprofile). diff --git a/docs/reference/utility-apis/OAuthApi.md b/docs/reference/utility-apis/OAuthApi.md new file mode 100644 index 0000000000..e73d5f645f --- /dev/null +++ b/docs/reference/utility-apis/OAuthApi.md @@ -0,0 +1,121 @@ +# OAuthApi + +The OAuthApi type is defined at +[packages/core-api/src/apis/definitions/auth.ts:67](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L67). + +The following Utility APIs implement this type: + +- [githubAuthApiRef](./README.md#githubauth) + +- [gitlabAuthApiRef](./README.md#gitlabauth) + +- [googleAuthApiRef](./README.md#googleauth) + +- [microsoftAuthApiRef](./README.md#microsoftauth) + +- [oauth2ApiRef](./README.md#oauth2) + +- [oktaAuthApiRef](./README.md#oktaauth) + +## Members + +### getAccessToken() + +Requests an OAuth 2 Access Token, optionally with a set of scopes. The access +token allows you to make requests on behalf of the user, and the copes may grant +you broader access, depending on the auth provider. + +Each auth provider has separate handling of scope, so you need to look at the +documentation for each one to know what scope you need to request. + +This method is cheap and should be called each time an access token is used. Do +not for example store the access token in React component state, as that could +cause the token to expire. Instead fetch a new access token for each request. + +Be sure to include all required scopes when requesting an access token. When +testing your implementation it is best to log out the Backstage session and then +visit your plugin page directly, as you might already have some required scopes +in your existing session. Not requesting the correct scopes can lead to 403 or +other authorization errors, which can be tricky to debug. + +If the user has not yet granted access to the provider and the set of requested +scopes, the user will be prompted to log in. The returned promise will not +resolve until the user has successfully logged in. The returned promise can be +rejected, but only if the user rejects the login request. + +
+getAccessToken(
+    scope?: OAuthScope,
+    options?: AuthRequestOptions,
+  ): Promise<string>
+
+ +### logout() + +Log out the user's session. This will reload the page. + +
+logout(): Promise<void>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AuthRequestOptions + +
+export type AuthRequestOptions = {
+  /**
+   * If this is set to true, the user will not be prompted to log in,
+   * and an empty response will be returned if there is no existing session.
+   *
+   * This can be used to perform a check whether the user is logged in, or if you don't
+   * want to force a user to be logged in, but provide functionality if they already are.
+   *
+   * @default false
+   */
+  optional?: boolean;
+
+  /**
+   * If this is set to true, the request will bypass the regular oauth login modal
+   * and open the login popup directly.
+   *
+   * The method must be called synchronously from a user action for this to work in all browsers.
+   *
+   * @default false
+   */
+  instantPopup?: boolean;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:40](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L40). + +Referenced by: [getAccessToken](#getaccesstoken). + +### OAuthScope + +This file contains declarations for common interfaces of auth-related APIs. The +declarations should be used to signal which type of authentication and +authorization methods each separate auth provider supports. + +For example, a Google OAuth provider that supports OAuth 2 and OpenID Connect, +would be declared as follows: + +const googleAuthApiRef = createApiRef({ ... }) + +An array of scopes, or a scope string formatted according to the auth provider, +which is typically a space separated list. + +See the documentation for each auth provider for the list of scopes supported by +each provider. + +
+export type OAuthScope = string | string[]
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:38](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L38). + +Referenced by: [getAccessToken](#getaccesstoken). diff --git a/docs/reference/utility-apis/OAuthRequestApi.md b/docs/reference/utility-apis/OAuthRequestApi.md new file mode 100644 index 0000000000..c6b9e09189 --- /dev/null +++ b/docs/reference/utility-apis/OAuthRequestApi.md @@ -0,0 +1,232 @@ +# OAuthRequestApi + +The OAuthRequestApi type is defined at +[packages/core-api/src/apis/definitions/OAuthRequestApi.ts:99](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L99). + +The following Utility API implements this type: +[oauthRequestApiRef](./README.md#oauthrequest) + +## Members + +### createAuthRequester() + +A utility for showing login popups or similar things, and merging together +multiple requests for different scopes into one request that inclues all scopes. + +The passed in options provide information about the login provider, and how to +handle auth requests. + +The returned AuthRequester function is used to request login with new scopes. +These requests are merged together and forwarded to the auth handler, as soon as +a consumer of auth requests triggers an auth flow. + +See AuthRequesterOptions, AuthRequester, and handleAuthRequests for more info. + +
+createAuthRequester<AuthResponse>(
+    options: AuthRequesterOptions<AuthResponse>,
+  ): AuthRequester<AuthResponse>
+
+ +### authRequest\$() + +Observers panding auth requests. The returned observable will emit all current +active auth request, at most one for each created auth requester. + +Each request has its own info about the login provider, forwarded from the auth +requester options. + +Depending on user interaction, the request should either be rejected, or used to +trigger the auth handler. If the request is rejected, all pending AuthRequester +calls will fail with a "RejectedError". If a auth is triggered, and the auth +handler resolves successfully, then all currently pending AuthRequester calls +will resolve to the value returned by the onAuthRequest call. + +
+authRequest$(): Observable<PendingAuthRequest[]>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AuthProvider + +Information about the auth provider that we're requesting a login towards. + +This should be shown to the user so that they can be informed about what login +is being requested before a popup is shown. + +
+export type AuthProvider = {
+  /**
+   * Title for the auth provider, for example "GitHub"
+   */
+  title: string;
+
+  /**
+   * Icon for the auth provider.
+   */
+  icon: IconComponent;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/OAuthRequestApi.ts:27](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L27). + +Referenced by: [AuthRequesterOptions](#authrequesteroptions), +[PendingAuthRequest](#pendingauthrequest). + +### AuthRequester + +Function used to trigger new auth requests for a set of scopes. + +The returned promise will resolve to the same value returned by the +onAuthRequest in the AuthRequesterOptions. Or rejected, if the request is +rejected. + +This function can be called multiple times before the promise resolves. All +calls will be merged into one request, and the scopes forwarded to the +onAuthRequest will be the union of all requested scopes. + +
+export type AuthRequester<AuthResponse> = (
+  scopes: Set<string>,
+) => Promise<AuthResponse>
+
+ +Defined at +[packages/core-api/src/apis/definitions/OAuthRequestApi.ts:66](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L66). + +Referenced by: [createAuthRequester](#createauthrequester). + +### AuthRequesterOptions + +Describes how to handle auth requests. Both how to show them to the user, and +what to do when the user accesses the auth request. + +
+export type AuthRequesterOptions<AuthResponse> = {
+  /**
+   * Information about the auth provider, which will be forwarded to auth requests.
+   */
+  provider: AuthProvider;
+
+  /**
+   * Implementation of the auth flow, which will be called synchronously when
+   * trigger() is called on an auth requests.
+   */
+  onAuthRequest(scopes: Set<string>): Promise<AuthResponse>;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/OAuthRequestApi.ts:43](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L43). + +Referenced by: [createAuthRequester](#createauthrequester). + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [authRequest\$](#authrequest). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### PendingAuthRequest + +An pending auth request for a single auth provider. The request will remain in +this pending state until either reject() or trigger() is called. + +Any new requests for the same provider are merged into the existing pending +request, meaning there will only ever be a single pending request for a given +provider. + +
+export type PendingAuthRequest = {
+  /**
+   * Information about the auth provider, as given in the AuthRequesterOptions
+   */
+  provider: AuthProvider;
+
+  /**
+   * Rejects the request, causing all pending AuthRequester calls to fail with "RejectedError".
+   */
+  reject: () => void;
+
+  /**
+   * Trigger the auth request to continue the auth flow, by for example showing a popup.
+   *
+   * Synchronously calls onAuthRequest with all scope currently in the request.
+   */
+  trigger(): Promise<void>;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/OAuthRequestApi.ts:77](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L77). + +Referenced by: [authRequest\$](#authrequest). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/reference/utility-apis/OpenIdConnectApi.md b/docs/reference/utility-apis/OpenIdConnectApi.md new file mode 100644 index 0000000000..41a3247af6 --- /dev/null +++ b/docs/reference/utility-apis/OpenIdConnectApi.md @@ -0,0 +1,79 @@ +# OpenIdConnectApi + +The OpenIdConnectApi type is defined at +[packages/core-api/src/apis/definitions/auth.ts:104](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L104). + +The following Utility APIs implement this type: + +- [auth0AuthApiRef](./README.md#auth0auth) + +- [googleAuthApiRef](./README.md#googleauth) + +- [microsoftAuthApiRef](./README.md#microsoftauth) + +- [oauth2ApiRef](./README.md#oauth2) + +- [oktaAuthApiRef](./README.md#oktaauth) + +## Members + +### getIdToken() + +Requests an OpenID Connect ID Token. + +This method is cheap and should be called each time an ID token is used. Do not +for example store the id token in React component state, as that could cause the +token to expire. Instead fetch a new id token for each request. + +If the user has not yet logged in to Google inside Backstage, the user will be +prompted to log in. The returned promise will not resolve until the user has +successfully logged in. The returned promise can be rejected, but only if the +user rejects the login request. + +
+getIdToken(options?: AuthRequestOptions): Promise<string>
+
+ +### logout() + +Log out the user's session. This will reload the page. + +
+logout(): Promise<void>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AuthRequestOptions + +
+export type AuthRequestOptions = {
+  /**
+   * If this is set to true, the user will not be prompted to log in,
+   * and an empty response will be returned if there is no existing session.
+   *
+   * This can be used to perform a check whether the user is logged in, or if you don't
+   * want to force a user to be logged in, but provide functionality if they already are.
+   *
+   * @default false
+   */
+  optional?: boolean;
+
+  /**
+   * If this is set to true, the request will bypass the regular oauth login modal
+   * and open the login popup directly.
+   *
+   * The method must be called synchronously from a user action for this to work in all browsers.
+   *
+   * @default false
+   */
+  instantPopup?: boolean;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:40](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L40). + +Referenced by: [getIdToken](#getidtoken). diff --git a/docs/reference/utility-apis/ProfileInfoApi.md b/docs/reference/utility-apis/ProfileInfoApi.md new file mode 100644 index 0000000000..09c0f88f83 --- /dev/null +++ b/docs/reference/utility-apis/ProfileInfoApi.md @@ -0,0 +1,98 @@ +# ProfileInfoApi + +The ProfileInfoApi type is defined at +[packages/core-api/src/apis/definitions/auth.ts:127](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L127). + +The following Utility APIs implement this type: + +- [auth0AuthApiRef](./README.md#auth0auth) + +- [githubAuthApiRef](./README.md#githubauth) + +- [gitlabAuthApiRef](./README.md#gitlabauth) + +- [googleAuthApiRef](./README.md#googleauth) + +- [microsoftAuthApiRef](./README.md#microsoftauth) + +- [oauth2ApiRef](./README.md#oauth2) + +- [oktaAuthApiRef](./README.md#oktaauth) + +## Members + +### getProfile() + +Get profile information for the user as supplied by this auth provider. + +If the optional flag is not set, a session is guaranteed to be returned, while +if the optional flag is set, the session may be undefined. See +@AuthRequestOptions for more details. + +
+getProfile(options?: AuthRequestOptions): Promise<ProfileInfo | undefined>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### AuthRequestOptions + +
+export type AuthRequestOptions = {
+  /**
+   * If this is set to true, the user will not be prompted to log in,
+   * and an empty response will be returned if there is no existing session.
+   *
+   * This can be used to perform a check whether the user is logged in, or if you don't
+   * want to force a user to be logged in, but provide functionality if they already are.
+   *
+   * @default false
+   */
+  optional?: boolean;
+
+  /**
+   * If this is set to true, the request will bypass the regular oauth login modal
+   * and open the login popup directly.
+   *
+   * The method must be called synchronously from a user action for this to work in all browsers.
+   *
+   * @default false
+   */
+  instantPopup?: boolean;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:40](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L40). + +Referenced by: [getProfile](#getprofile). + +### ProfileInfo + +Profile information of the user. + +
+export type ProfileInfo = {
+  /**
+   * Email ID.
+   */
+  email?: string;
+
+  /**
+   * Display name that can be presented to the user.
+   */
+  displayName?: string;
+
+  /**
+   * URL to an avatar image of the user.
+   */
+  picture?: string;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:172](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L172). + +Referenced by: [getProfile](#getprofile). diff --git a/docs/reference/utility-apis/README.md b/docs/reference/utility-apis/README.md new file mode 100644 index 0000000000..cfdc3b6ef8 --- /dev/null +++ b/docs/reference/utility-apis/README.md @@ -0,0 +1,173 @@ +# Backstage Core Utility APIs + +The following is a list of all Utility APIs defined by `@backstage/core`. They +are available to use by plugins and components, and can be accessed using the +`useApi` hook, also provided by `@backstage/core`. For more information, see +https://github.com/spotify/backstage/blob/master/docs/api/utility-apis.md. + +### alert + +Used to report alerts and forward them to the app + +Implemented type: [AlertApi](./AlertApi.md) + +ApiRef: +[alertApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AlertApi.ts#L41) + +### appTheme + +API Used to configure the app theme, and enumerate options + +Implemented type: [AppThemeApi](./AppThemeApi.md) + +ApiRef: +[appThemeApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/AppThemeApi.ts#L74) + +### auth0Auth + +Provides authentication towards Auth0 APIs + +Implemented types: [OpenIdConnectApi](./OpenIdConnectApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[auth0AuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L273) + +### config + +Used to access runtime configuration + +Implemented type: [Config](./Config.md) + +ApiRef: +[configApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/ConfigApi.ts#L22) + +### discovery + +Provides service discovery of backend plugins + +Implemented type: [DiscoveryApi](./DiscoveryApi.md) + +ApiRef: +[discoveryApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/DiscoveryApi.ts#L44) + +### error + +Used to report errors and forward them to the app + +Implemented type: [ErrorApi](./ErrorApi.md) + +ApiRef: +[errorApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/ErrorApi.ts#L65) + +### featureFlags + +Used to toggle functionality in features across Backstage + +Implemented type: [FeatureFlagsApi](./FeatureFlagsApi.md) + +ApiRef: +[featureFlagsApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/FeatureFlagsApi.ts#L58) + +### githubAuth + +Provides authentication towards GitHub APIs + +Implemented types: [OAuthApi](./OAuthApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[githubAuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L230) + +### gitlabAuth + +Provides authentication towards GitLab APIs + +Implemented types: [OAuthApi](./OAuthApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[gitlabAuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L260) + +### googleAuth + +Provides authentication towards Google APIs and identities + +Implemented types: [OAuthApi](./OAuthApi.md), +[OpenIdConnectApi](./OpenIdConnectApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[googleAuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L213) + +### identity + +Provides access to the identity of the signed in user + +Implemented type: [IdentityApi](./IdentityApi.md) + +ApiRef: +[identityApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/IdentityApi.ts#L54) + +### microsoftAuth + +Provides authentication towards Microsoft APIs and identities + +Implemented types: [OAuthApi](./OAuthApi.md), +[OpenIdConnectApi](./OpenIdConnectApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[microsoftAuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L287) + +### oauth2 + +Example of how to use oauth2 custom provider + +Implemented types: [OAuthApi](./OAuthApi.md), +[OpenIdConnectApi](./OpenIdConnectApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), [SessionStateApi](./SessionStateApi.md) + +ApiRef: +[oauth2ApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L301) + +### oauthRequest + +An API for implementing unified OAuth flows in Backstage + +Implemented type: [OAuthRequestApi](./OAuthRequestApi.md) + +ApiRef: +[oauthRequestApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/OAuthRequestApi.ts#L130) + +### oktaAuth + +Provides authentication towards Okta APIs + +Implemented types: [OAuthApi](./OAuthApi.md), +[OpenIdConnectApi](./OpenIdConnectApi.md), +[ProfileInfoApi](./ProfileInfoApi.md), +[BackstageIdentityApi](./BackstageIdentityApi.md), +[SessionStateApi](./SessionStateApi.md) + +ApiRef: +[oktaAuthApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L243) + +### storage + +Provides the ability to store data which is unique to the user + +Implemented type: [StorageApi](./StorageApi.md) + +ApiRef: +[storageApiRef](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/StorageApi.ts#L68) diff --git a/docs/reference/utility-apis/SessionStateApi.md b/docs/reference/utility-apis/SessionStateApi.md new file mode 100644 index 0000000000..c1821f2381 --- /dev/null +++ b/docs/reference/utility-apis/SessionStateApi.md @@ -0,0 +1,119 @@ +# SessionStateApi + +The SessionStateApi type is defined at +[packages/core-api/src/apis/definitions/auth.ts:201](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L201). + +The following Utility APIs implement this type: + +- [auth0AuthApiRef](./README.md#auth0auth) + +- [githubAuthApiRef](./README.md#githubauth) + +- [gitlabAuthApiRef](./README.md#gitlabauth) + +- [googleAuthApiRef](./README.md#googleauth) + +- [microsoftAuthApiRef](./README.md#microsoftauth) + +- [oauth2ApiRef](./README.md#oauth2) + +- [oktaAuthApiRef](./README.md#oktaauth) + +## Members + +### sessionState\$() + +
+sessionState$(): Observable<SessionState>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [sessionState\$](#sessionstate). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### SessionState + +Session state values passed to subscribers of the SessionStateApi. + +
+export enum SessionState {
+  SignedIn = 'SignedIn',
+  SignedOut = 'SignedOut',
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/auth.ts:192](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/auth.ts#L192). + +Referenced by: [sessionState\$](#sessionstate). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/reference/utility-apis/StorageApi.md b/docs/reference/utility-apis/StorageApi.md new file mode 100644 index 0000000000..bee52935da --- /dev/null +++ b/docs/reference/utility-apis/StorageApi.md @@ -0,0 +1,186 @@ +# StorageApi + +The StorageApi type is defined at +[packages/core-api/src/apis/definitions/StorageApi.ts:31](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/StorageApi.ts#L31). + +The following Utility API implements this type: +[storageApiRef](./README.md#storage) + +## Members + +### forBucket() + +Create a bucket to store data in. + +
+forBucket(name: string): StorageApi
+
+ +### get() + +Get the current value for persistent data, use observe\$ to be notified of +updates. + +
+get<T>(key: string): T | undefined
+
+ +### remove() + +Remove persistent data. + +
+remove(key: string): Promise<void>
+
+ +### set() + +Save persistant data, and emit messages to anyone that is using observe\$ for +this key + +
+set(key: string, data: any): Promise<void>
+
+ +### observe\$() + +Observe changes on a particular key in the bucket + +
+observe$<T>(key: string): Observable<StorageValueChange<T>>
+
+ +## Supporting types + +These types are part of the API declaration, but may not be unique to this API. + +### Observable + +Observable sequence of values and errors, see TC39. + +https://github.com/tc39/proposal-observable + +This is used as a common return type for observable values and can be created +using many different observable implementations, such as zen-observable or +RxJS 5. + +
+export type Observable<T> = {
+  /**
+   * Subscribes to this observable to start receiving new values.
+   */
+  subscribe(observer: Observer<T>): Subscription;
+  subscribe(
+    onNext: (value: T) => void,
+    onError?: (error: Error) => void,
+    onComplete?: () => void,
+  ): Subscription;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:53](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L53). + +Referenced by: [observe\$](#observe), [StorageApi](#storageapi). + +### Observer + +This file contains non-react related core types used throught Backstage. + +Observer interface for consuming an Observer, see TC39. + +
+export type Observer<T> = {
+  next?(value: T): void;
+  error?(error: Error): void;
+  complete?(): void;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:24](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L24). + +Referenced by: [Observable](#observable). + +### StorageApi + +
+export interface StorageApi {
+  /**
+   * Create a bucket to store data in.
+   * @param {String} name Namespace for the storage to be stored under,
+   *                      will inherit previous namespaces too
+   */
+  forBucket(name: string): StorageApi;
+
+  /**
+   * Get the current value for persistent data, use observe$ to be notified of updates.
+   *
+   * @param {String} key Unique key associated with the data.
+   * @return {Object} data The data that should is stored.
+   */
+  get<T>(key: string): T | undefined;
+
+  /**
+   * Remove persistent data.
+   *
+   * @param {String} key Unique key associated with the data.
+   */
+  remove(key: string): Promise<void>;
+
+  /**
+   * Save persistant data, and emit messages to anyone that is using observe$ for this key
+   *
+   * @param {String} key Unique key associated with the data.
+   */
+  set(key: string, data: any): Promise<void>;
+
+  /**
+   * Observe changes on a particular key in the bucket
+   * @param {String} key Unique key associated with the data
+   */
+  observe$<T>(key: string): Observable<StorageValueChange<T>>;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/StorageApi.ts:31](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/StorageApi.ts#L31). + +Referenced by: [forBucket](#forbucket). + +### StorageValueChange + +
+export type StorageValueChange<T = any> = {
+  key: string;
+  newValue?: T;
+}
+
+ +Defined at +[packages/core-api/src/apis/definitions/StorageApi.ts:21](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/apis/definitions/StorageApi.ts#L21). + +Referenced by: [observe\$](#observe), [StorageApi](#storageapi). + +### Subscription + +Subscription returned when subscribing to an Observable, see TC39. + +
+export type Subscription = {
+  /**
+   * Cancels the subscription
+   */
+  unsubscribe(): void;
+
+  /**
+   * Value indicating whether the subscription is closed.
+   */
+  readonly closed: Boolean;
+}
+
+ +Defined at +[packages/core-api/src/types.ts:33](https://github.com/spotify/backstage/blob/82d329555c16af46db9b4e5cd2f44a3cc006a52e/packages/core-api/src/types.ts#L33). + +Referenced by: [Observable](#observable). diff --git a/docs/journey.md b/docs/tutorials/journey.md similarity index 94% rename from docs/journey.md rename to docs/tutorials/journey.md index e38130afb7..24765e8531 100644 --- a/docs/journey.md +++ b/docs/tutorials/journey.md @@ -1,12 +1,16 @@ -# Purpose +--- +id: journey +title: Future developer journey +description: This document describes a possible journey of a future Backstage +--- -This RFC describes a possible journey of a future Backstage plugin developer as -they build a plugin that touches many different aspects of a Backstage. The -story invents many new things that are not part of Backstage today, but are -things that I'm suggesting we should add as long term or north star goals. The -idea is to discuss what parts of the story makes sense to aim for, and what we'd -want to do differently or not at all. The "chapters" are numbered to make it a -bit easier to comment on parts of the story. +> This document describes a possible journey of a **_future_** Backstage plugin +> developer as they build a plugin that touches many different aspects of a +> Backstage. The story invents many new things that are not part of Backstage +> today, but are things that I'm suggesting we should add as long term or north +> star goals. The idea is to discuss what parts of the story makes sense to aim +> for, and what we'd want to do differently or not at all. The "chapters" are +> numbered to make it a bit easier to comment on parts of the story. # The Protagonist diff --git a/docs/tutorials/quickstart-app-auth.md b/docs/tutorials/quickstart-app-auth.md new file mode 100644 index 0000000000..2615e37633 --- /dev/null +++ b/docs/tutorials/quickstart-app-auth.md @@ -0,0 +1,199 @@ +--- +id: quickstart-app-auth +title: Monorepo App Setup With Authentication +--- + +###### September 15th 2020 - @backstage/create-app - v0.1.1-alpha.21 + +
+ +> This document takes you through setting up a backstage app that runs in your +> own environment. It starts with a skeleton install and verifying of the +> monorepo's functionality. Next, GitHub authentication is added and tested. +> +> This document assumes you have NodeJS 12 active along with Yarn. Please note, +> that at the time of this writing, the current version is 0.1.1-alpha.21. This +> guide can still be used with future versions, just, verify as you go. If you +> run into issues, you can compare your setup with mine here > +> [simple-backstage-app](https://github.com/johnson-jesse/simple-backstage-app). + +# The Skeleton Application + +From the terminal: + +1. Create a (monorepo) application: `npx @backstage/create-app` +1. Enter an `id` for your new app like `mybiz-backstage` I went with + `simple-backstage-app` +1. Choose `SQLite` as your database. This is the quickest way to get started as + PostgreSQL requires additional setup not covered here. +1. Start your backend: `yarn --cwd packages/backend start` + +```zsh +# You should see positive verbiage in your terminal output +2020-09-11T22:20:26.712Z backstage info Listening on :7000 +``` + +5. Finally, start the frontend. Open a new terminal window and from the root of + your project, run: `yarn start` + +```zsh +# You should see positive verbiage in your terminal output +ℹ ｢wds｣: Project is running at http://localhost:3000/ +``` + +Once the app compiles, a browser window should have popped with your stand-alone +application loaded at `localhost:3000`. This could take a couple minutes. + +```zsh +# You should see positive verbiage in your terminal output +ℹℹ ｢wdm｣: Compiled successfully. +``` + +Since there is no auth currently configured, you are automatically entered as a +guest. Let's fix that now and add auth. + +# The Auth Configuration + +1. Open `app-config.yaml` and change it as follows + +_from:_ + +```yaml +auth: + providers: {} +``` + +_to:_ + +```yaml +auth: + providers: + github: + development: + clientId: + $secret: + env: AUTH_GITHUB_CLIENT_ID + clientSecret: + $secret: + env: AUTH_GITHUB_CLIENT_SECRET + ## uncomment the following three lines if using enterprise + # enterpriseInstanceUrl: + # $secret: + # env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL +``` + +2. Set environment variables in whatever fashion is easiest for you. I chose to + add mine to my `.zshrc` profile. + +```zsh +# For macOS Catalina & Z Shell +# ------ simple-backstage-app GitHub +export AUTH_GITHUB_CLIENT_ID=xxx +export AUTH_GITHUB_CLIENT_SECRET=xxx +# export AUTH_GITHUB_ENTERPRISE_INSTANCE_URL=https://github.{MY_BIZ}.com +``` + +3. And of course I need to source that file. + +```zsh +# Loading the new variables +% source ~/.zshrc + +# Any other currently opened terminals need to be restarted to pick up the new values +# verify your setup by running env +% env +# should output something like +> ... +> AUTH_GITHUB_CLIENT_ID=xxx +> AUTH_GITHUB_CLIENT_SECRET=xxx +> ... +``` + +4. The values to replace `xxx` above come from your oauth app setup. + +``` +> Log into http://github.com +> Navigate to (Settings > Developer Settings > OAuth Apps > New OAuth App)[https://github.com/settings/applications/new] +> Set Homepage URL = http://localhost:3000 +> Set Callback URL = http://localhost:7000/auth/github +> Click [Register application] +> On the next page, copy and paste your new Client ID and Client Secret to the environment variables above, `AUTH_GITHUB_CLIENT_ID` & `AUTH_GITHUB_CLIENT_SECRET` +> Don't forget to `source` that profile file again if necessary. +``` + +5. Open and change _root > packages > app > src >_`App.tsx` as follows + +```tsx +// Add the following imports to the existing list from core +import { githubAuthApiRef, SignInPage } from '@backstage/core'; +``` + +6. In the same file, change the createApp function as follows + +```tsx +const app = createApp({ + apis, + plugins: Object.values(plugins), + components: { + SignInPage: props => { + return ( + + ); + }, + }, +}); +``` + +6. Open and change _root > packages > app > src >_ `apis.ts` as follows + +```ts +// Add the following imports to the existing list from core +import { githubAuthApiRef, GithubAuth } from '@backstage/core'; +``` + +7. In the same file, change the builder block for oauthRequestApiRef as follows + +_from:_ + +```ts +builder.add(oauthRequestApiRef, new OAuthRequestManager()); +``` + +_to:_ + +```ts +const oauthRequestApi = builder.add( + oauthRequestApiRef, + new OAuthRequestManager(), +); + +builder.add( + githubAuthApiRef, + GithubAuth.create({ + discoveryApi, + oauthRequestApi, + }), +); +``` + +> Start the backend and frontend as before. When the browser loads, you should +> be presented with a login page for GitHub. Login as usual with your GitHub +> account. If this is your first time, you will be asked to authorize and then +> are redirected to the catalog page if all is well. + +# Where to go from here + +> You're probably eager to write your first custom plugin. Follow this next +> tutorial for an in-depth look at a custom GitHub repository browser plugin. +> [Adding Custom Plugin to Existing Monorepo App](quickstart-app-plugin.md). diff --git a/docs/tutorials/quickstart-app-plugin.md b/docs/tutorials/quickstart-app-plugin.md new file mode 100644 index 0000000000..28efb2a007 --- /dev/null +++ b/docs/tutorials/quickstart-app-plugin.md @@ -0,0 +1,487 @@ +--- +id: quickstart-app-plugin +title: Adding Custom Plugin to Existing Monorepo App +--- + +###### September 15th 2020 - v0.1.1-alpha.21 + +
+ +> This document takes you through setting up a new plugin for your existing +> monorepo with a _GitHub provider already setup_. If you don't have either of +> those, you can clone +> [simple-backstage-app](https://github.com/johnson-jesse/simple-backstage-app) +> which this document builds on. +> +> This document does not cover authoring a plugin for sharing with the Backstage +> community. That will have to be a later discussion. +> +> We start with a skeleton plugin install. And after verifying its +> functionality, extend the Sidebar to make our life easy. Finally, we add +> custom code to display GitHub repository information. +> +> This document assumes you have NodeJS 12 active along with Yarn. Please note, +> that at the time of this writing, the current version is 0.1.1-alpha.21. This +> guide can still be used with future versions, just, verify as you go. If you +> run into issues, you can compare your setup with mine here > +> [simple-backstage-app-plugin](https://github.com/johnson-jesse/simple-backstage-app-plugin). + +# The Skeleton Plugin + +1. Start by using the built in creator. From the terminal and root of your + project run: `yarn create-plugin` +1. Enter a plugin ID. I used `github-playground` +1. When the process finishes, let's start the backend: + `yarn --cwd packages/backend start` +1. If you see errors starting, refer to + [Auth Configuration](https://github.com/johnson-jesse/simple-backstage-app/blob/master/README.md#the-auth-configuration) + for more information on environment variables. +1. And now the frontend, from a new terminal window and the root of your + project: `yarn start` +1. As usual, a browser window should popup loading the App. +1. Now manually navigate to our plugin page from your browser: + `http://localhost:3000/github-playground` +1. You should see successful verbiage for this endpoint, + `Welcome to github-playground!` + +# The Shortcut + +Let's add a shortcut. + +1. Open and modify `root: packages > app > src > sidebar.tsx` with the + following: + +```tsx +import GitHubIcon from '@material-ui/icons/GitHub'; +... + +``` + +Simple! The App will reload with your changes automatically. You should now see +a github icon displayed in the sidebar. Clicking that will link to our new +plugin. And now, the API fun begins. + +# The Identity + +Our first modification will be to extract information from the Identity API. + +1. Start by opening + `root: plugins > github-playground > src > components > ExampleComponent > ExampleComponent.tsx` +1. Add two new imports + +```tsx +// Add identityApiRef to the list of imported from core +import { identityApiRef } from '@backstage/core'; +import { useApi } from '@backstage/core-api'; +``` + +3. Adjust the ExampleComponent from inline to block + +_from inline:_ + +```tsx +const ExampleComponent: FC<{}> = () => ( ... ) +``` + +_to block:_ + +```tsx +const ExampleComponent: FC<{}> = () => { + + return ( + ... + ) +} +``` + +4. Now add our hook and const data before the return statement + +```tsx +// our API hook +const identityApi = useApi(identityApiRef); + +// data to use +const userId = identityApi.getUserId(); +const profile = identityApi.getProfile(); +``` + +5. Finally, update the InfoCard's jsx to use our new data + +```tsx + + + {`${profile.displayName} | ${profile.email}`} + + +``` + +If everything is saved, you should see your name, id, and email on the +github-playground page. Our data accessed is synchronous. So we just grab and +go. + +6. Here is the entire file for reference +
Complete ExampleComponent.tsx +

+ +```tsx +import React, { FC } from 'react'; +import { Typography, Grid } from '@material-ui/core'; +import { + InfoCard, + Header, + Page, + pageTheme, + Content, + ContentHeader, + HeaderLabel, + SupportButton, + identityApiRef, +} from '@backstage/core'; +import { useApi } from '@backstage/core-api'; +import ExampleFetchComponent from '../ExampleFetchComponent'; + +const ExampleComponent: FC<{}> = () => { + const identityApi = useApi(identityApiRef); + const userId = identityApi.getUserId(); + const profile = identityApi.getProfile(); + + return ( + +

+ + +
+ + + A description of your plugin goes here. + + + + + + {`${profile.displayName} | ${profile.email}`} + + + + + + + + + + ); +}; + +export default ExampleComponent; +``` + +

+
+ +# The Wipe + +The last file we will touch is ExampleFetchComponent. Because of the number of +changes, let's start by wiping this component clean. + +1. Start by opening + `root: plugins > github-playground > src > components > ExampleFetchComponent > ExampleFetchComponent.tsx` +1. Replace everyting in the file with the following: + +```tsx +import React, { FC } from 'react'; +import { useAsync } from 'react-use'; +import Alert from '@material-ui/lab/Alert'; +import { + Table, + TableColumn, + Progress, + githubAuthApiRef, +} from '@backstage/core'; +import { useApi } from '@backstage/core-api'; +import { graphql } from '@octokit/graphql'; + +const ExampleFetchComponent: FC<{}> = () => { + return
Nothing to see yet
; +}; + +export default ExampleFetchComponent; +``` + +3. Save that and ensure you see no errors. Comment out the unused imports if + your linter gets in the way. + +###### We will add a lot to this file for the sake of ease. Please don't do this in productional code! + +# The Graph Model + +GitHub has a graphql API available for interacting. Let's start by adding our +basic repository query + +1. Add the query const statement outside ExampleFetchComponent + +```tsx +const query = `{ + viewer { + repositories(first: 100) { + totalCount + nodes { + name + createdAt + description + diskUsage + isFork + } + pageInfo { + endCursor + hasNextPage + } + } + } +}`; +``` + +2. Using this structure as a guide, we will break our query into type parts +3. Add the following outside of ExampleFetchComponent + +```tsx +type Node = { + name: string; + createdAt: string; + description: string; + diskUsage: number; + isFork: boolean; +}; + +type Viewer = { + repositories: { + totalCount: number; + nodes: Node[]; + pageInfo: { + endCursor: string; + hasNextPage: boolean; + }; + }; +}; +``` + +# The Tabel Model + +Using Backstage's own component library, let's define a custom table. This +component will get used if we have data to display. + +1. Add the following outside of ExampleFetchComponent + +```tsx +type DenseTableProps = { + viewer: Viewer; +}; + +export const DenseTable: FC = ({ viewer }) => { + const columns: TableColumn[] = [ + { title: 'Name', field: 'name' }, + { title: 'Created', field: 'createdAt' }, + { title: 'Description', field: 'description' }, + { title: 'Disk Usage', field: 'diskUsage' }, + { title: 'Fork', field: 'isFork' }, + ]; + + return ( + + ); +}; +``` + +# The Fetch + +We're ready to flush out our fetch component + +1. Add our api hook inside ExampleFetchComponent + +```tsx +const auth = useApi(githubAuthApiRef); +``` + +2. The access token we need to make our GitHub request and the request itself is + obtained in an asynchronous manner. +3. Add the useAsync block inside the ExampleFetchComponent + +```tsx +const { value, loading, error } = useAsync(async (): Promise => { + const token = await auth.getAccessToken(); + + const gqlEndpoint = graphql.defaults({ + // Uncomment baseUrl if using enterprise + // baseUrl: 'https://github.MY-BIZ.com/api', + headers: { + authorization: `token ${token}`, + }, + }); + const { viewer } = await gqlEndpoint(query); + return viewer; +}, []); +``` + +4. The resolved data is conventiently destructured with value containing our + Viewer type. loading as a boolean, self explainatory. And error which is + present only if necessary. So let's use those as the first 3 of 4 multi + return statements. +5. Add the _if return_ blocks below our async block + +```tsx +if (loading) return ; +if (error) return {error.message}; +if (value && value.repositories) return ; +``` + +6. The third line here utilizes our custom table accepting our Viewer type. +7. Finally, we add our _else return_ block to catch any other scenarios. + +```tsx +return ( +
+); +``` + +8. After saving that, and given we don't have any errors, you should see a table + with basic information on your repositories. +9. Here is the entire file for reference +
Complete ExampleFetchComponent.tsx +

+ +```tsx +import React, { FC } from 'react'; +import { useAsync } from 'react-use'; +import Alert from '@material-ui/lab/Alert'; +import { + Table, + TableColumn, + Progress, + githubAuthApiRef, +} from '@backstage/core'; +import { useApi } from '@backstage/core-api'; +import { graphql } from '@octokit/graphql'; + +const query = `{ +viewer { + repositories(first: 100) { + totalCount + nodes { + name + createdAt + description + diskUsage + isFork + } + pageInfo { + endCursor + hasNextPage + } + } +} +}`; + +type Node = { + name: string; + createdAt: string; + description: string; + diskUsage: number; + isFork: boolean; +}; + +type Viewer = { + repositories: { + totalCount: number; + nodes: Node[]; + pageInfo: { + endCursor: string; + hasNextPage: boolean; + }; + }; +}; + +type DenseTableProps = { + viewer: Viewer; +}; + +export const DenseTable: FC = ({ viewer }) => { + const columns: TableColumn[] = [ + { title: 'Name', field: 'name' }, + { title: 'Created', field: 'createdAt' }, + { title: 'Description', field: 'description' }, + { title: 'Disk Usage', field: 'diskUsage' }, + { title: 'Fork', field: 'isFork' }, + ]; + + return ( +

+ ); +}; + +const ExampleFetchComponent: FC<{}> = () => { + const auth = useApi(githubAuthApiRef); + + const { value, loading, error } = useAsync(async (): Promise => { + const token = await auth.getAccessToken(); + + const gqlEndpoint = graphql.defaults({ + // Uncomment baseUrl if using enterprise + // baseUrl: 'https://github.MY-BIZ.com/api', + headers: { + authorization: `token ${token}`, + }, + }); + const { viewer } = await gqlEndpoint(query); + return viewer; + }, []); + + if (loading) return ; + if (error) return {error.message}; + if (value && value.repositories) return ; + + return ( +
+ ); +}; + +export default ExampleFetchComponent; +``` + +

+ + +10. We finished! If there are no errors, you should see your own GitHub + repoistory information displayed in a basic table. If you run into issues, + you can compare the repo that backs this documdnt, + [simple-backstage-app-plugin](https://github.com/johnson-jesse/simple-backstage-app-plugin) + +# Where to go from here + +> Break apart ExampleFetchComponent into smaller logical parts contained in +> their own files. Rename your components to something other than ExampleXxx. +> +> You might be real proud of a plugin you develop. Follow this next tutorial for +> an in-depth look at publishing and including that for the entire Backstage +> community. [TODO](#). diff --git a/docs/verify-links.js b/docs/verify-links.js new file mode 100755 index 0000000000..9bf8a67ad8 --- /dev/null +++ b/docs/verify-links.js @@ -0,0 +1,131 @@ +#!/usr/bin/env node +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { resolve: resolvePath, dirname } = require('path'); +const fs = require('fs-extra'); +const recursive = require('recursive-readdir'); + +const projectRoot = resolvePath(__dirname, '..'); + +async function verifyUrl(basePath, url) { + // Avoid having absolute URL links within docs/, so that links work on the site + if ( + url.match( + /https:\/\/github.com\/spotify\/backstage\/(tree|blob)\/master\/docs\//, + ) && + basePath.match(/^(?:docs|microsite)\//) + ) { + return { url, basePath, problem: 'absolute' }; + } + + url = url.replace(/#.*$/, ''); + url = url.replace( + /https:\/\/github.com\/spotify\/backstage\/(tree|blob)\/master/, + '', + ); + if (!url) { + return; + } + + // Only verify existence of local files for now, so skip anything with a schema + if (url.match(/[a-z]+:/)) { + return; + } + + let path = ''; + + if (url.startsWith('/')) { + if (url.startsWith('/docs/') && basePath.match(/^(?:docs|microsite)\//)) { + return { url, basePath, problem: 'not-relative' }; + } + + const staticPath = resolvePath(projectRoot, 'microsite/static', `.${url}`); + if (await fs.pathExists(staticPath)) { + return; + } + + path = resolvePath(projectRoot, `.${url}`); + } else { + path = resolvePath(dirname(resolvePath(projectRoot, basePath)), url); + } + + const exists = await fs.pathExists(path); + if (!exists) { + return { url, basePath, problem: 'missing' }; + } + + return; +} + +async function verifyFile(filePath) { + const content = await fs.readFile(filePath, 'utf8'); + const mdLinks = content.match(/\[.+?\]\(.+?\)/g) || []; + const badUrls = []; + + for (const mdLink of mdLinks) { + const url = mdLink.match(/\[.+\]\((.+)\)/)[1].trim(); + const badUrl = await verifyUrl(filePath, url); + if (badUrl) { + badUrls.push(badUrl); + } + } + + return badUrls; +} + +async function main() { + process.chdir(projectRoot); + + const files = await recursive('.', [ + 'node_modules', + 'dist', + 'bin', + 'microsite', + ]); + const mdFiles = files.filter(f => f.endsWith('.md')); + const badUrls = []; + + for (const mdFile of mdFiles) { + const badFileUrls = await verifyFile(mdFile); + badUrls.push(...badFileUrls); + } + + if (badUrls.length) { + console.log(`Found ${badUrls.length} bad links within repo`); + for (const { url, basePath, problem } of badUrls) { + if (problem === 'missing') { + console.error( + `Unable to reach ${url} from root or microsite/static/, linked from ${basePath}`, + ); + } else if (problem === 'not-relative') { + console.error('Links to /docs/ must be relative'); + console.error(` From: ${basePath}`); + console.error(` To: ${url}`); + } else if (problem === 'absolute') { + console.error(`Link to docs/ should be replaced by a relative URL`); + console.error(` From: ${basePath}`); + console.error(` To: ${url}`); + } + } + process.exit(1); + } +} + +main().catch(error => { + console.error(error.stack); + process.exit(1); +}); diff --git a/install/kubernetes/ingress.yaml b/install/kubernetes/ingress.yaml deleted file mode 100644 index d2ecae74c6..0000000000 --- a/install/kubernetes/ingress.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: backstage - labels: - app: backstage - component: ingress -spec: - rules: - - host: - http: - paths: - - backend: - serviceName: backstage - servicePort: frontend - path: / - - backend: - serviceName: backstage-backend - servicePort: backend - path: /backend diff --git a/lerna.json b/lerna.json index d5a6d08c6c..a018b1a289 100644 --- a/lerna.json +++ b/lerna.json @@ -2,5 +2,5 @@ "packages": ["packages/*", "plugins/*"], "npmClient": "yarn", "useWorkspaces": true, - "version": "0.1.1-alpha.15" + "version": "0.1.1-alpha.22" } diff --git a/microsite/.gitignore b/microsite/.gitignore new file mode 100644 index 0000000000..f5bcdb3385 --- /dev/null +++ b/microsite/.gitignore @@ -0,0 +1,4 @@ + +# Build output +build +i18n diff --git a/microsite/README.md b/microsite/README.md new file mode 100644 index 0000000000..d244caa22a --- /dev/null +++ b/microsite/README.md @@ -0,0 +1,206 @@ +This website was created with [Docusaurus](https://docusaurus.io/). + +# What's In This Document + +- [Getting Started](#getting-started) +- [Directory Structure](#directory-structure) +- [Editing Content](#editing-content) +- [Adding Content](#adding-content) +- [Full Documentation](#full-documentation) + +# Getting Started + +## Installation + +``` +$ yarn install +``` + +## Local Development + +``` +$ yarn start +``` + +This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server. + +## Build + +``` +$ yarn build +``` + +This command generates static content into the `build` directory, which is what will be deployed to GitHub pages from the master branch. + +## Directory Structure + +Your project file structure should look something like this + +``` +my-docusaurus/ + docs/ + doc-1.md + doc-2.md + doc-3.md + website/ + blog/ + 2016-3-11-oldest-post.md + 2017-10-24-newest-post.md + core/ + node_modules/ + pages/ + static/ + css/ + img/ + package.json + sidebars.json + siteConfig.js +``` + +# Editing Content + +## Editing an existing docs page + +Edit docs by navigating to `docs/` and editing the corresponding document: + +`docs/doc-to-be-edited.md` + +```markdown +--- +id: page-needs-edit +title: This Doc Needs To Be Edited +--- + +Edit me... +``` + +For more information about docs, click [here](https://docusaurus.io/docs/en/navigation) + +## Editing an existing blog post + +Edit blog posts by navigating to `website/blog` and editing the corresponding post: + +`website/blog/post-to-be-edited.md` + +```markdown +--- +id: post-needs-edit +title: This Blog Post Needs To Be Edited +--- + +Edit me... +``` + +For more information about blog posts, click [here](https://docusaurus.io/docs/en/adding-blog) + +# Adding Content + +## Adding a new docs page to an existing sidebar + +1. Create the doc as a new markdown file in `/docs`, example `docs/newly-created-doc.md`: + +```md +--- +id: newly-created-doc +title: This Doc Needs To Be Edited +--- + +My new content here.. +``` + +1. Refer to that doc's ID in an existing sidebar in `website/sidebars.json`: + +```javascript +// Add newly-created-doc to the Getting Started category of docs +{ + "docs": { + "Getting Started": [ + "quick-start", + "newly-created-doc" // new doc here + ], + ... + }, + ... +} +``` + +For more information about adding new docs, click [here](https://docusaurus.io/docs/en/navigation) + +## Adding a new blog post + +1. Make sure there is a header link to your blog in `website/siteConfig.js`: + +`website/siteConfig.js` + +```javascript +headerLinks: [ + ... + { blog: true, label: 'Blog' }, + ... +] +``` + +2. Create the blog post with the format `YYYY-MM-DD-My-Blog-Post-Title.md` in `website/blog`: + +`website/blog/2018-05-21-New-Blog-Post.md` + +```markdown +--- +author: Frank Li +authorURL: https://twitter.com/foobarbaz +authorFBID: 503283835 +title: New Blog Post +--- + +Lorem Ipsum... +``` + +For more information about blog posts, click [here](https://docusaurus.io/docs/en/adding-blog) + +## Adding items to your site's top navigation bar + +1. Add links to docs, custom pages or external links by editing the headerLinks field of `website/siteConfig.js`: + +`website/siteConfig.js` + +```javascript +{ + headerLinks: [ + ... + /* you can add docs */ + { doc: 'my-examples', label: 'Examples' }, + /* you can add custom pages */ + { page: 'help', label: 'Help' }, + /* you can add external links */ + { href: 'https://github.com/facebook/docusaurus', label: 'GitHub' }, + ... + ], + ... +} +``` + +For more information about the navigation bar, click [here](https://docusaurus.io/docs/en/navigation) + +## Adding custom pages + +1. Docusaurus uses React components to build pages. The components are saved as .js files in `website/pages/en`: +1. If you want your page to show up in your navigation header, you will need to update `website/siteConfig.js` to add to the `headerLinks` element: + +`website/siteConfig.js` + +```javascript +{ + headerLinks: [ + ... + { page: 'my-new-custom-page', label: 'My New Custom Page' }, + ... + ], + ... +} +``` + +For more information about custom pages, click [here](https://docusaurus.io/docs/en/custom-pages). + +# Full Documentation + +Full documentation can be found on the [website](https://docusaurus.io/). diff --git a/microsite/blog/2020-03-16-announcing-backstage.md b/microsite/blog/2020-03-16-announcing-backstage.md new file mode 100644 index 0000000000..3911debf08 --- /dev/null +++ b/microsite/blog/2020-03-16-announcing-backstage.md @@ -0,0 +1,44 @@ +--- +title: Announcing Backstage +author: Stefan Ålund +authorURL: http://twitter.com/stalund +authorImageURL: https://pbs.twimg.com/profile_images/121166861/6919c047c0d0edaace78c3009b28e917-user-full-200-130.generated_400x400.jpg +--- + +## What is Backstage? + +Backstage is Spotify's open source platform for building developer portals. + +It’s the first open source infrastructure platform by Spotify that allows you to focus on building your application instead of reinventing the button. With an elegant and unified, yet opinionated UI/UX for all your tooling and infrastructure, Backstage enables engineers to get up and running faster, which ultimately makes their lives easier and more productive. + +![img](assets/blog_1.png) + + + +## As simple as writing a plugin. + +Backstage makes it easy to unify all of your infrastructure tooling, services, and documentation under a single, easy-to-use interface. So your engineers will always know where to find the right tool for the job. And engineers will already know how to use each tool — because everything uses the same, familiar UI. + +The number of open source infrastructure projects and tools [landscape](https://landscape.cncf.io/) is exploding. As the sheer volume of projects increases, companies and their engineers find it increasingly difficult to keep track and adopt all of the tooling fast enough to keep pace. Most of the tools were built by a different individual, team, or company, which means that there is no single UI/UX, and simply getting the tool installed and started can be a painful challenge- let alone wrangling each tool to work with one another within your existing ecosystem. Due to varying qualities and the varying UI/UX of each open source project, we'd like to introduce Backstage as a best-of-breed platform for developers to use... all in service of ensuring a flawless, consistent user experience. + +![illustration](assets/illustration.svg) + +## The Spotify story + +A best-in-class developer portal — from a music company? Since the very beginning, Spotify has been known for its agile, autonomous engineering culture. More than music, we’re a tech company that has always put engineers first, empowering our developers with the ability to innovate quickly and at scale. Backstage is the natural result of that focus. + +Since adopting Backstage internally at Spotify, we’ve seen a 55% decrease in onboarding time for our engineers (as measured by time until 10th pull request). Over 280 engineering teams inside Spotify are using Backstage to manage 2,000+ backend services, 300+ websites, 4,000+ data pipelines, and 200+ mobile features. + +## Project roadmap + +We created Backstage about 4 years ago, and today, we’ve decided to share the goodness with the greater engineering community. While our version of Backstage has had the benefit of time to mature and evolve, the first iteration of our open source version is still nascent. I wanted to take a moment to share with you what our vision for Backstage OSS is so that 1. users and our community gain a better understanding of where we’re envisioning the product to go and more importantly, 2. you can provide input and feedback so that together, we can create a better infrastructure experience for developers everywhere. + +We are envisioning three phases of the project and we have already begun work on various aspects of these phases: + +- **Phase 1:** Extensible frontend platform (now) - You will be able to easily create a single consistent UI layer for your internal infrastructure and tools. A set of reusable UX patterns and components help ensure a consistent experience between tools. + +- **Phase 2:** Manage your stuff (next 2-3 months) - Manage anything from microservices to software components to infrastructure and your service catalog. Regardless of whether you want to create a new library, view service deployment status in Kubernetes, or check the test coverage for a website -- Backstage will provide all of those tools - and many more - in a single developer portal. + +- **Phase 3:** Ecosystem (later) - Everyone's infrastructure stack is different. By fostering a vibrant community of contributors we hope to provide an ecosystem of Open Source plugins/integrations that allows you to pick the tools that match your stack. + +Our vision for Backstage is for it to become the trusted standard toolbox (read: UI layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. If this sounds interesting or you'd like to help us shape our product vision, we'd love to talk. You can email me directly: [alund@spotify.com](mailto:alund@spotify.com). diff --git a/microsite/blog/2020-03-18-what-is-backstage.md b/microsite/blog/2020-03-18-what-is-backstage.md new file mode 100644 index 0000000000..8b0a0fcbe6 --- /dev/null +++ b/microsite/blog/2020-03-18-what-is-backstage.md @@ -0,0 +1,88 @@ +--- +title: What the heck is Backstage anyway? +author: Stefan Ålund +authorURL: http://twitter.com/stalund +authorImageURL: https://pbs.twimg.com/profile_images/121166861/6919c047c0d0edaace78c3009b28e917-user-full-200-130.generated_400x400.jpg +--- + +![img](assets/2/spotify-labs-header.png) + +Two days ago, we released the open source version of [Backstage](https://backstage.io/), our homegrown developer portal. And we learned a thing or two via the feedback we received. So, I wanted to take this opportunity to further explain what we’re trying to do with Backstage — and more importantly, what we want to give to the greater engineering community beyond Spotify. + + + +## What’s the big infrastructure problem? + +As companies grow, their infrastructure systems get messier. Consider a team that wants to deploy something to the cloud. While Spotify has many awesome engineers, not every engineer is well-versed in our chosen cloud-provider tooling. Yet everyone is required to know and understand Terraform, GCP/AWS/Azure CLIs, GitLab CI, Prometheus, Kubernetes, Docker, various monitoring and alerting tools, and much, much more. Once other resources come into play (databases, queueing, etc.), each engineer requires even more tools and domain-specific knowledge (or “disciplines”), from backend to machine learning, to mobile and data. + +## What’s the fix? + +Backstage unifies all your infrastructure tooling, services, and documentation with a single, consistent UI. All of it! Imagine if all your tools — GCP, Bigtable, CI pipelines, TensorFlow Extended, and whatever else is hiding in your stack — all had the same, easy-to-use interface. That’s Backstage. One front end for all your infrastructure. + +![img](assets/2/screen.gif) + +Backstage gives developers a uniform overview of all their resources, regardless of how and where they are running, as well as an easy way to onboard and start using those tools. It also allows the creation of new resources, such as backend services running in Kubernetes, with a few clicks of a button — all without having to leave the same, familiar interface of Backstage. + +## Why did we build it? + +To some observers, it may seem odd that a music company is launching a best-in-class developer portal. But if you [dig deeper](https://backstage.io/background), you’ll find that since the very beginning, Spotify has been known for its agile, autonomous engineering culture. More than music, we’re a tech company that has always put engineers first, empowering our developers with the ability to innovate quickly and at scale. Backstage is the natural result of that focus. + +## What are examples of how Backstage is used at Spotify? + +Our internal installation of Backstage has over 100 different integrations — we call them “plugins”. Since the open-source version currently does not have any end-to-end use cases, it can be challenging to understand what problems Backstage can solve for you. To make things more tangible, let’s have a look at four of the common use-cases: + +1. Creating a new microservice +2. Following a pull request from review to production +3. Centralised technical documentation +4. Review performance of your team’s mobile features + +These are just a few examples. Expect us to continue providing examples of how Backstage is used inside Spotify while we build out more end-2-end use-cases in the open. + +### 1. Creating a new microservice + +Creating any new software component at Spotify, such as a new microservice, is done with a few clicks in Backstage. Developers choose between a number of standard templates — all with best-practices built in. + +![img](assets/2/1.png) + +After inputting some metadata about your service, a new repository is created with a “hello world” service that automatically builds and deploys in production on Kubernetes ([GKE](https://cloud.google.com/kubernetes-engine)). Ownership information is automatically captured in our service/software catalog and users can see a list of all the services they own. + +![img](assets/2/2.png) + +### 2. Following a pull request from review to production + +As soon as you submit a pull request to Spotify’s GitHub Enterprise, our CI system automatically posts a link to the CI/CD view in Backstage. The view provides you with all the information you need: build progress, test coverage changes, a re-trigger button, etc., so that you don’t have to look for this information across different systems. + +![img](assets/2/3.png) + +Our homegrown CI system uses Jenkins under the hood, but Spotify engineers don’t need to know that. They interact directly with GitHub Enterprise and Backstage. + +### 3. Centralised technical documentation + +Spotify uses a [docs-like-code](https://www.youtube.com/watch?v=uFGCaZmA6d4) approach. Engineers write technical documentation in Markdown files that live together with the code. During CI, a beautiful-looking documentation site is created using [MkDocs](https://www.mkdocs.org/), and all sites are rendered centrally in a Backstage plugin. + +![img](assets/2/4.png) + +On top of the static documentation we also incorporate additional metadata about the documentation site — such as owner, open issue and related Stack Overflow tags. + +### 4. Review performance of your team’s mobile features + +Our mobile apps are developed by many different teams. The codebase is divided up into different features, each owned and maintained by a separate team. If an app developer on one team wants to understand how their feature is affecting overall app performance, there’s a plugin for that: + +![img](assets/2/5.png) +_Figures above for illustrative purposes only._ + +Developers can also look at crashes, releases, test coverage over time and many more tools in the same location. + +## Why did we make Backstage open source? + +When discussing infrastructure challenges with peer companies, it’s clear that we are not alone in struggling with fragmentation across our developer ecosystem. As companies adopt more open-source tooling, and build more infrastructure internally, the complexity grows. It gets harder for individual engineers to find and use all these distinct tools. + +Similar to how Backstage ties together all of Spotify’s infrastructure, our ambition is to make the open-source version of Backstage the standard UX layer across the broader infrastructure landscape. We decided to release Backstage early so we could collaborate more closely with companies that have a similar problem — and that want to provide a better developer experience to their teams. + +## What’s next? + +We are envisioning [three phases](https://github.com/spotify/backstage/milestones) of the project (so far), and we have already begun work on various aspects of these phases. The best way to track the work and see where you can jump in and help out is: + +https://github.com/spotify/backstage/milestones + +Want to discuss the project or need support? Join us on [Discord](https://discord.gg/MUpMjP2) or reach out on [alund@spotify.com](mailto:alund@spotify.com). diff --git a/microsite/blog/2020-04-06-lighthouse-plugin.md b/microsite/blog/2020-04-06-lighthouse-plugin.md new file mode 100644 index 0000000000..2d5c7b96b7 --- /dev/null +++ b/microsite/blog/2020-04-06-lighthouse-plugin.md @@ -0,0 +1,40 @@ +--- +title: Introducing Lighthouse for Backstage +author: Paul Marbach +authorURL: http://twitter.com/fastfrwrd +authorImageURL: https://pbs.twimg.com/profile_images/1224058798958088192/JPxS8uzR_400x400.jpg +--- + +![image illustrating the Lighthouse plugin for Backstage](assets/3/lead.png) + +We’re proud to announce that our first internal plugin at Spotify has been open-sourced as part of Backstage. This plugin works with the newly open-sourced [lighthouse-audit-service](https://github.com/spotify/lighthouse-audit-service) to run and track Lighthouse audits for your websites. + + + +## What is Lighthouse? + +Google's [Lighthouse](https://developers.google.com/web/tools/lighthouse) auditing tool for websites is a great open-source resource for benchmarking and improving the accessibility, performance, SEO, and best practices of your site. + +At Spotify, we keep track of Lighthouse audit scores over time to look at trends and areas for investment. We particularly look to Lighthouse to give us [accessibility recommendations](https://developers.google.com/web/tools/lighthouse/v3/scoring#a11y); in the next few months, we plan to roll out Lighthouse accessibility category scores as a benchmark metric for all websites at Spotify. + +## Lighthouse, tracked over time + +What makes the plugin unique is that we can track a website's audit performance over time using the main metrics that Lighthouse outputs, rather than simply running reports. The sparklines show, at a glance, how all of your websites are holding up over recent builds. + +![image of the audit list in the Lighthouse plugin](assets/3/audit-list.png) + +Lighthouse reports can be viewed directly in Backstage, with the ability to travel back and forth through your audit history, so you can quickly diagnose which release caused a performance or SEO regression. + +![image of the audit view in the Lighthouse plugin](assets/3/audit-view.png) + +Trigger an audit directly from Backstage, or trigger audits programmatically with your new lighthouse-audit-service instance. Schedule them after builds as a sort of smoke test, or trigger them on a schedule (as we do at Spotify) to get a daily snapshot of your website. + +![image of the create audit form in the Lighthouse plugin](assets/3/create-audit.png) + +## Using Lighthouse in Backstage + +To learn how you can enable Lighthouse auditing within Backstage, head over to the [README](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) for the plugin to get started. + +## A personal note + +I want to thank the folks on the Backstage team for approaching me to open-source this plugin. I have found working on Backstage to be a really rewarding and fun time, and I'm so glad that the core team members have put in the effort to make Backstage something that anyone in the industry can use. I can't wait to play with all the plugins the community is going to create. I am hopeful that this plugin can help illustrate just a sliver of what we use Backstage for at Spotify. diff --git a/microsite/blog/2020-04-30-how-to-quickly-set-up-backstage.md b/microsite/blog/2020-04-30-how-to-quickly-set-up-backstage.md new file mode 100644 index 0000000000..dff567eaf3 --- /dev/null +++ b/microsite/blog/2020-04-30-how-to-quickly-set-up-backstage.md @@ -0,0 +1,135 @@ +--- +title: How to quickly set up Backstage +author: Marcus Eide +authorURL: https://github.com/marcuseide +authorImageURL: https://secure.gravatar.com/avatar/20223f1e03673c7c1e6282fbebaf6942 +--- + +We wanted to make getting started with Backstage as easy as possible. Even though Backstage is still in the early phases of its development, we believe it is important for our users to get a feel for what Backstage really is. + +We want users to be able to create their own version of Backstage quickly and easily, so that they can take advantage of all the infrastructure that we’ve built into it — and start exploring. + +In this blog post we’ll look at what a Backstage app is and how to create one using our [CLI](https://www.npmjs.com/package/@backstage/cli). + + + +## What is a Backstage app? + +![](assets/4/welcome.png) + +A Backstage app is a modern monorepo web project that is built using Backstage packages. It includes all the configuration and architecture you need to run Backstage so that you don’t have to worry about setting everything up by yourself. + +More specifically, a Backstage app includes the core packages and APIs that provide base functionality to the app. The actual UX is provided by plugins. As an example, when you first load the `/` page of the app, the content is provided by the `welcome` plugin. + +Plugins are the essential building blocks of Backstage and extend the platform by providing additional features and functionality. Read more about [Backstage plugins](/docs/getting-started) on GitHub. + +## A personalized platform + +When you create a Backstage _app_, you are creating your own installation of Backstage, an application that is built on top of the Backstage _platform_. + +You get to take full advantage of a platform that we at Spotify have been using internally for years. But you also get to make it your own — starting with its name. You can rename the Backstage app anything you want, so that you can call it something that best fits your organization. Be creative! + +## How do I create an app? + +Just run the backstage-cli: + +```bash +npx @backstage/create-app +``` + +Name your app, and we will create everything you need: + +![](assets/4/create-app.png) + +The only thing you need to do is to start the app: + +```bash +cd my-app +yarn start +``` + +And you are good to go! 👍 + +Read the full documentation on how to [create an app](/docs/getting-started/create-an-app) on GitHub. + +## What do I get? (Let's get technical...) + +We’ve been using Backstage internally for years, and we’ve spent a lot of time adding to and tweaking the infrastructure so that it fits our needs. After all that testing and trial and error, we think it will fit your needs, too! + +### 1. Lerna setup to manage multi-packages + +The monorepo and its packages are managed by [Lerna](https://lerna.js.org/). It lets you work with individual packages in a controlled way. + +### 2. Fast builds + +Behind the scenes we use [Rollup](https://rollupjs.org/) to build the modules. + +Each package is built individually. With the `--watch` flag you will be able to detect changes per package and therefore speed up the local development process. + +To further speed things up, we have also included our own caching system to avoid rebuilding unchanged packages. + +Our hope is that there will be thousands of Backstage plugins in the future, so we need a fast and stable build process. + +### 3. Full TypeScript support + +Most of the codebase is written in [TypeScript](https://www.typescriptlang.org/), and we aim for all of the core packages to be in TypeScript in the future. + +All the knobs and handles needed for a stable and functioning TypeScript project are included. + +Take a look at `@backstage/cli/config/tsconfig.json` for more details. + +### 4. Tests and coverage out of the box + +We include testing, linting, and end-to-end tests for your convenience. + +```bash +yarn lint:all +yarn test:all +yarn test:e2e +``` + +## Extend the app with plugins + +At Spotify, the main factor behind Backstage’s success has been our large and diverse collection of plugins — the result of contributions from various teams over the years. Internally, we have more than a hundred different plugins. + +There are two ways to add plugins to your Backstage app: use a publicly available plugin or create your own. + +### Using a public plugin + +We provide a collection of public Backstage plugins (look for packages with the `plugin-` prefix under the `@backstage` namespace on [npm](https://www.npmjs.com/) that you can start using immediately. + +Install in your app’s package folder (`/packages/app`) with: + +```bash +yarn add @backstage/plugin- +``` + +Then add it to your app's `plugin.ts` file to import and register it: + +`/packages/app/src/plugin.ts`: + +```js +export { plugin as PluginName } from '@backstage/plugin-'; +``` + +A plugin registers its own `route` in the app — read the documentation for the specific plugin you are installing for more information on that. + +### Creating an internal plugin + +We also know that each organization has different needs and will create their own plugins for internal purposes. To create an internal plugin, you can use our CLI again. + +In the root of your app directory (``) run: + +```bash +yarn create-plugin +``` + +This command will create a new plugin in `/plugins/` and register it to your app automatically. + +### Sharing is caring 🤗 + +If you are developing a plugin that might be useful for others, consider releasing it publicly. A large, diverse ecosystem of Backstage plugins benefits the whole community + +## Ready to get started? + +Head over to GitHub and check out the [project](https://github.com/spotify/backstage) or download our [CLI](https://www.npmjs.com/package/@backstage/cli). If you have more questions, join us on [Discord](https://discord.gg/MUpMjP2) or [create an issue](https://github.com/spotify/backstage/issues/new/choose). diff --git a/microsite/blog/2020-05-14-tech-radar-plugin.md b/microsite/blog/2020-05-14-tech-radar-plugin.md new file mode 100644 index 0000000000..d78d5d67fa --- /dev/null +++ b/microsite/blog/2020-05-14-tech-radar-plugin.md @@ -0,0 +1,44 @@ +--- +title: Introducing Tech Radar for Backstage +author: Bilawal Hameed +authorURL: http://twitter.com/bilawalhameed +authorImageURL: https://avatars0.githubusercontent.com/bih +--- + +![image illustrating the Tech Radar plugin for Backstage](assets/5/lead.png) + +Just a few weeks ago, we released our internal plugin for [Lighthouse website audits] as our first open source plugin, so the whole community could use it. Today, we’re excited to add a new plugin to that list — say hello to the [Tech Radar plugin]! + + + +## What is Tech Radar? + +The Technology Radar is a concept created by [ThoughtWorks] which allows you to visualize the official guidelines of software languages, processes, infrastructure, and platforms at that particular company. The particular visualization above was created by [Zalando]. + +At Spotify, our central committee of technical architects own the Tech Radar with the input of engineers across the company. Anyone can and is encouraged to give recommendations. We segment entries in our Tech Radar by languages, frameworks, processes, and infrastructure, although you should pick whatever works best for your organization. Each entry in the Tech Radar can have one of the following lifecycle values: Use, Trial, Assess, and Hold. + +We also assign clear definitions for each lifecycle: + +- **Use:** This technology is recommended for use by the majority of teams with a specific use case. +- **Trial:** This technology has been evaluated for specific use cases and has showed clear benefits. Some teams adopt it in production, although it should be limited to low-impact projects as it might incur a higher risk. +- **Assess:** This technology has the potential to be beneficial for the company. Some teams are evaluating it and using it in experimental projects. Using it in production comes with a high cost and risk due to lack of in-house knowledge, maintenance, and support. +- **Hold:** We don’t want to further invest in this technology or we evaluated it and we don’t see it as beneficial for the company. Teams should not use it in new projects and should plan on migrating to a supported alternative if they use it for historical reasons. For broadly adopted technologies, the Radar should refer to a migration path to a supported alternative. + +Since rolling out the Tech Radar, it has become the source of truth when creating, maintaining, or evolving our software ecosystem. Spotify has dozens of entries in our Radar and it can scale quite well whilst being easy for our engineers and engineering managers to consume. + +## Using the Tech Radar in Backstage + +To learn about how you can bring the Tech Radar to your Backstage installation, check out [the plugin README on GitHub][tech radar plugin]. + +## A personal note + +I want to thank both the Backstage team and Spotify. Firstly, I’ve been working with our internal version of Backstage for over a year, and the developer experience since open sourcing has been even more of a joy to work with. Secondly, the 10% hack time that Spotify generously provides to all engineers enabled me to open source the Tech Radar plugin. + +Since open sourcing it, the community has shown great interest in yet another powerful use case of Backstage. There was also an enthusiastic open source contributor who volunteered to migrate the plugin to TypeScript and React Hooks [in just 29 minutes](https://github.com/spotify/backstage/issues/661) of opening the issue! + +I can’t wait to see how others benefit from the Tech Radar in their organizations! + +[lighthouse website audits]: https://backstage.io/blog/2020/04/06/lighthouse-plugin +[tech radar plugin]: https://github.com/spotify/backstage/tree/master/plugins/tech-radar +[thoughtworks]: https://www.thoughtworks.com/radar +[zalando]: https://opensource.zalando.com/tech-radar/ diff --git a/microsite/blog/2020-05-14-weaveworks-covid-19-app-uses-backstage-ui.md b/microsite/blog/2020-05-14-weaveworks-covid-19-app-uses-backstage-ui.md new file mode 100644 index 0000000000..68e3f903c7 --- /dev/null +++ b/microsite/blog/2020-05-14-weaveworks-covid-19-app-uses-backstage-ui.md @@ -0,0 +1,33 @@ +--- +title: Weaveworks’ COVID-19 app uses Backstage UI +author: Jeff Feng +authorURL: https://github.com/fengypants +authorImageURL: https://avatars2.githubusercontent.com/u/46946747 +--- + +![fk-covid-screenshot](assets/20-05-14/weaveworks-firekube-covid-19-spotify-backstage.png) + +One of the great things about the open source community is once you put your work out there, you really never know where it might end up. That’s certainly the case here. + + + +When Weaveworks decided to build an X-ray diagnostics app to help fight COVID-19, they pulled together a lot of different open source resources — from projects big and small, built by both familiar tech partners and some brand new ones, too. + +At the heart of their app — called [fk-covid][] — there’s a TensorFlow-based deep neural network that was developed by researchers on the DarwinAI team and others in the COVID R&D community. To package that network up for doctors and software developers to use, the app combines open source tools from Google, AWS, Azure, MinIO, the CNCF, and Weaveworks’ own Firekube bundle for Kubernetes. + +And the user interface for all of this? Weaveworks built a custom plugin using the Backstage framework. + +“We chose Backstage as a modern UI toolkit that we knew would work with Kubernetes apps,” said Alexis Richardson, CEO of Weaveworks. “We were also experimenting with Backstage for microservices and ML, so it was natural to try it here.” + +Chanwit Kaewkasi, Weaveworks’ DX Engineer and a tech lead on the project, said, “Backstage offers very advanced plugin architecture which allows us to only focus on the plugin we're developing. Other things are taken care of by the framework.” + +In other words, here’s Backstage doing what Backstage does best: unifying a bunch of technologies with a cohesive frontend, so that the whole thing is easier to build and easier to use. + +Joining the fight against a global pandemic was not something the Backstage team at Spotify ever envisioned when we released our homegrown developer portal to the world back in March. But it’s a testament to the ingenuity (and serendipity) of the open source community that Backstage could be enlisted for such an unexpected use case. + +We’re proud to see Backstage adopted as the UX layer for this meaningful cause. And we can’t wait to see what the open source community will build next. + +To learn more about what fk-covid does, and how it works, jump on over to [the Weaveworks blog][] to hear it straight from the team that built it. It’s a great example of the possibilities that come from being a part of the open source community. + +[fk-covid]: https://github.com/weaveworks/fk-covid +[the weaveworks blog]: https://www.weave.works/blog/firekube-covid-ml diff --git a/microsite/blog/2020-05-22-phase-2-service-catalog.md b/microsite/blog/2020-05-22-phase-2-service-catalog.md new file mode 100644 index 0000000000..682965d2dd --- /dev/null +++ b/microsite/blog/2020-05-22-phase-2-service-catalog.md @@ -0,0 +1,54 @@ +--- +title: Starting Phase 2: The Service Catalog +author: Stefan Ålund +authorURL: http://twitter.com/stalund +authorImageURL: https://pbs.twimg.com/profile_images/121166861/6919c047c0d0edaace78c3009b28e917-user-full-200-130.generated_400x400.jpg +--- + +**TL;DR** Thanks to the help from the Backstage community, we’ve made excellent progress and are now moving into Phase 2 of Backstage — building out a Service Catalog and the surrounding systems that will help unify the tools you use to manage your software. + +We released the open source version of Backstage a little less than two months ago, and have been thrilled to see so many people jumping in and contributing to the project in its early stages. We’re excited to see what the community can build together as we progress through [each phase of Backstage](https://github.com/spotify/backstage#project-roadmap). + +![img](assets/20-05-20/Service_Catalog_MVP.png) + + + +## Progress so far + +Phase 1 was all about building an extensible frontend platform, enabling teams to start creating a single, consistent UI layer for your internal infrastructure and tools in the form of [plugins](https://github.com/spotify/backstage/labels/plugin). In fact, thanks to our amazing (30+) [contributors](https://github.com/spotify/backstage/graphs/contributors), we were able to complete most of Phase 1 earlier than expected. 🎉 + +Today, we are happy to announce that we are shifting our focus to Phase 2! + +## So what is Phase 2? + +> _The core of building Platforms rests in versatile entity management. Entities represent the nouns or the "truths" of our world._ + +Quote from [Platform Nuts & Bolts: Extendable Data Models](https://www.kislayverma.com/post/platform-nuts-bolts-extendable-data-models) + +Entities, or what we refer to as “components” in Backstage, represent all software, including services, websites, libraries, data pipelines, and so forth. The focus of Phase 2 will be on adding an entity model in Backstage that makes it easy for engineers to create and manage the software components they own. + +With the ability to create a plethora of components in Backstage, how does one keep track of all the software in the ecosystem? Therein lies the highlight feature of Phase 2: the [Service Catalog](https://github.com/spotify/backstage/milestone/4). The service catalog — or software catalog — is a centralized system that keeps track of ownership and metadata about all software in your ecosystem. The catalog is built around the concept of [metadata yaml files](/docs/architecture-decisions/adr002-default-catalog-file-format.md) stored together with the code, which are then harvested and visualized in Backstage. + +![img](assets/20-05-20/Service_Catalog_MVP.png) + +![img](assets/20-05-20/Service_Catalog_MVP_service.png) + +With a single catalog, Backstage makes it easy for a team to manage ten services — and makes it possible for your company to manage thousands of them. Because the system is practically self-organizing, it requires hardly any oversight from a governing or centralized team. Developers can get a uniform overview of all their software and related resources (such as server utilisation, data pipelines, pull request status), regardless of how and where they are running, as well as an easy way to onboard and manage those resources. + +On top of that, we have found that the service catalog is a great way to organise the infrastructure tools you use to manage the software as well. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different infrastructure UI’s (and incurring additional cognitive overhead each time they make a context switch), most of these tools can be organised around the entities in the catalog: + +![img](assets/20-05-20/tabs.png) + +More concretely, having this structure in place will allow plugins such as [CircleCI](https://github.com/spotify/backstage/tree/master/plugins/circleci) to show only the builds for the specific service you are viewing, or a [Spinnaker](https://github.com/spotify/backstage/issues/631) plugin to show running deployments, or an Open API plugin to [show documentation](https://github.com/spotify/backstage/issues/627) for endpoints exposed by the service, or the [Lighthouse](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) plugin to show audit reports for your website. You get the point. + +## Timeline + +Our estimated timeline has us delivering these pieces in increments leading up to June 22. But with the support of the community we wouldn’t be surprised if things land earlier than that. 🙏 + +If you are interested in joining us, check out our [Milestones](https://github.com/spotify/backstage/milestones) and connected Issues. + +## Long-term vision + +Our vision for Backstage is for it to become the trusted, standard toolbox (read: UX layer) for the open source infrastructure landscape. Imagine a future where regardless of what infrastructure you use inside your company, there is an open source plugin available that you can pick up and add to your deployment of Backstage. + +Spotify will continue to release more of our [internal](https://backstage.io/blog/2020/04/06/lighthouse-plugin) [plugins](https://backstage.io/blog/2020/05/14/tech-radar-plugin), but participation from developers and companies can help us build a healthy community. We are excited to see how Backstage has helped many of you, and look forward to seeing all the new plugins you and your teams will build! diff --git a/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md b/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md new file mode 100644 index 0000000000..0928b8769b --- /dev/null +++ b/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md @@ -0,0 +1,54 @@ +--- +title: Backstage Service Catalog released in alpha +author: Stefan Ålund +authorURL: http://twitter.com/stalund +image: https://backstage.io/blog/assets/6/header.png +--- + +**TL;DR** Today we are announcing the availability of the Backstage Service Catalog in alpha. This has been the community’s most requested feature. Even if the catalog is not ready for production yet, we think this release already demonstrates how Backstage can provide value for your company right out of the box. With your early input and feedback, we hope to create a stronger generally available product. + +![img](assets/6/header.png) + + + +## You asked, we listened + +When we [released](https://backstage.io/blog/2020/03/16/announcing-backstage) Backstage as an open source project back in March, it didn’t have all of the features that our internal version of Backstage has today. One of the main reasons we pushed to release it, despite it being in such a nascent stage, was so that we could start building the next phase of Backstage around the community’s needs. We’ve had hours of conversations with so many of you — thank you to everyone who has jumped on a video call, attended one of our working sessions, or watched our [demo videos](https://backstage.io/demos) and provided feedback via [Discord](https://discord.com/invite/MUpMjP2). + +Today, we wanted to share what we’ve learned from talking with many of you at companies that have shown interest in adopting Backstage. Here it is in short: + +- The problem of scaling autonomous engineering organisations without creating too much complexity is not a unique problem to Spotify. +- The "extensible frontend platform" that we focused on in the first phase of the project is not the only thing you are looking for. + +With these insights we decided to re-focus our efforts towards the most requested feature: the Backstage Service Catalog. + +## What is the service catalog? + +The Backstage Service Catalog — actually, a software catalog, since it includes more than just services — is a centralized system that keeps track of ownership and metadata for all the software in your ecosystem (services, websites, libraries, data pipelines, etc). The catalog is built around the concept of [metadata yaml files](/docs/architecture-decisions/adr002-default-catalog-file-format.md#format) stored together with the code, which are then harvested and visualized in Backstage. + +This was our pitch for the virtues of a service catalog when we first [announced](https://backstage.io/blog/2020/05/22/phase-2-service-catalog) it as part of Phase 2: + +> With a single catalog, Backstage makes it easy for a team to manage ten services — and makes it possible for your company to manage thousands of them. Because the system is practically self-organizing, it requires hardly any oversight from a governing or centralized team. Developers can get a uniform overview of all their software and related resources (such as server utilisation, data pipelines, pull request status), regardless of how and where they are running, as well as an easy way to onboard and manage those resources. + +> On top of that, we have found that the service catalog is a great way to organise the infrastructure tools you use to manage the software as well. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different infrastructure UI’s (and incurring additional cognitive overhead each time they make a context switch), most of these tools can be organised around the entities in the catalog: + +![img](assets/20-05-20/tabs.png) + +You’ll be able to see many of these virtues in action with this alpha release — though, with some caveats, of course, since it is, after all, an alpha. + +## What does alpha mean? + +Alpha is our shorthand for "we don’t yet think Backstage is ready for production, but we’d love for you to test it and provide us with feedback". However, you should be able to try out the functionality of the service catalog: + +1. Register software components ([examples](https://github.com/spotify/backstage/tree/master/packages/catalog-model/examples)) +2. See all components represented in the catalog +3. Search across all components +4. Get an overview of the metadata of the components +5. Click through and get more information about a specific component (service, website, etc) +6. See example tooling (plugins) that helps you manage the component + +As with most alpha releases, you should expect things to change quite a lot until we reach the beta stage (we’re targeting the end of summer). There are obviously many things missing as well, but we wanted to start collecting feedback early and make it easier to see the end-to-end flow. + +If you have feedback or questions, please open a [GitHub issue](https://github.com/spotify/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send me an email at [alund@spotify.com](mailto:alund@spotify.com) 🙏 + +To get regular product updates and news about the Backstage community, sign up for the [Backstage newsletter](https://mailchi.mp/spotify/backstage-community). diff --git a/microsite/blog/2020-07-01-how-to-enable-authentication-in-backstage-using-passport.md b/microsite/blog/2020-07-01-how-to-enable-authentication-in-backstage-using-passport.md new file mode 100644 index 0000000000..ccdd8b34c6 --- /dev/null +++ b/microsite/blog/2020-07-01-how-to-enable-authentication-in-backstage-using-passport.md @@ -0,0 +1,46 @@ +--- +title: How to enable authentication in Backstage using Passport +author: Lee Mills +authorURL: https://github.com/leemills83 +authorImageURL: https://avatars1.githubusercontent.com/u/1236238?s=460&v=4 +--- + +![auth-landing-page](assets/20-07-01/auth-landing.png) + +Getting authentication right is important. It helps keep your platform safe, it’s one of the first things users will interact with, and there are many different authentication providers to support. To this end, we chose to use [Passport](http://www.passportjs.org/) to provide an easy-to-use, out-of-the-box experience that can be extended to your own, pre-existing authentication providers (known as strategies). The Auth APIs in Backstage serve two purposes: identify the user and provide a way for plugins to request access to third-party services on behalf of the user. We’ve already implemented Google and GitHub authentication to provide examples and to get you started. + + + +## What is Passport? + +[Passport](http://www.passportjs.org/) is Express-compatible authentication middleware for Node.js that provides access to over 500 authentication providers, covering everything from Google, Facebook, and Twitter to generic OAuth, SAML, and local. Check out all of the currently available [strategies listed on the Passport site](http://www.passportjs.org/). + +Passport has allowed us to leverage an existing open-source authentication framework that will, in turn, give users the freedom to add and extend alternative authentication strategies to their instance of Backstage. + +## Using authentication in Backstage + +![auth-landing-page](assets/20-07-01/auth-sidebar.png) + +First, check out the provided Google and GitHub implementations! [Spin up a local copy of Backstage](https://backstage.io/blog/2020/04/30/how-to-quickly-set-up-backstage) along with our example-backend. You can find more documentation on setting up the example backend [here](https://github.com/spotify/backstage/tree/master/packages/backend), but be sure to include the relevant client IDs and secrets when running `yarn start`: + +``` +AUTH_GOOGLE_CLIENT_ID=x AUTH_GOOGLE_CLIENT_SECRET=x AUTH_GITHUB_CLIENT_ID=x AUTH_GITHUB_CLIENT_SECRET=x SENTRY_TOKEN=x LOG_LEVEL=debug yarn start +``` + +You can find the implementation for these strategies along with a lightweight proof-of-concept implementation for SAML authentication at `/plugins/auth-backend/src/providers`. + +## Ready to get started by adding your chosen provider and implementation? + +Getting started is really straightforward, and can be broadly broken down into five steps: + +1. Install the [Passport-based provider package that best suits your needs](http://www.passportjs.org/). +2. Add a new provider to `plugins/auth-backend/src/providers/` +3. Implement the provider, extending the suitable framework, if needed. +4. Add the provider to the backend. +5. Add a frontend Auth Utility API. + +For full details, take a look at our [“Adding authentication providers” documentation](/docs/auth/add-auth-provider.md) and at the [excellent documentation](http://www.passportjs.org/docs/) provided by Passport. + +## Interested in contributing to the next steps for authentication? + +We’ve already seen both GitLab and Okta contributions from the community — and we’re thinking about a few more providers we’d like to add to Backstage, too. You can find those, and other authentication-related issues, in our repository by filtering with the [auth label](https://github.com/spotify/backstage/issues?q=is%3Aissue+is%3Aopen+label%3Aauth). diff --git a/microsite/blog/2020-08-05-announcing-backstage-software-templates.md b/microsite/blog/2020-08-05-announcing-backstage-software-templates.md new file mode 100644 index 0000000000..81d495e1c3 --- /dev/null +++ b/microsite/blog/2020-08-05-announcing-backstage-software-templates.md @@ -0,0 +1,78 @@ +--- +title: Announcing Backstage Software Templates +author: Stefan Ålund +authorURL: https://twitter.com/stalund +--- + +**TL;DR** Today we are announcing a new Backstage feature: Software Templates. Simplify setup, standardize tooling, and deploy with the click of a button. Using automated templates, your engineers can spin up a new microservice, website, or other software component with your organization’s best practices built-in, right from the start. + + + + + +## Balancing autonomy and standardization + +At Spotify, we’ve always believed in the speed and ingenuity that comes from having autonomous development teams. But as we learned firsthand, the faster you grow, the more fragmented and complex your software ecosystem becomes. And then everything slows down again. + +By centralizing services and standardizing your tooling, Backstage streamlines your development environment from end to end. Instead of restricting autonomy, standardization frees your engineers from infrastructure complexity. So you can return to building and scaling, quickly and safely. + +Today we are releasing one of the key features that helps balance autonomy and standardization: templates for creating software. + +## Backstage Software Templates: Push-button deployment + +Backstage Software Templates automate and standardize the process of creating software components. To show you how they work, we created four sample templates to get you started — just configure them to fit your tooling and off you go: + +- **Create React App Template** — create a new CRA website project +- **Golang Microservice** — create a Golang repo with this template built by members of the Go community +- **React SSR Template** — create a website powered with Next.js +- **Spring Boot GRPC** — create a simple microservice using gRPC and Spring Boot Java + +### The getting started guide gets automated + +Since the templates can be customized to integrate with your existing infrastructure, it’s easy to start a new project without ever having to leave Backstage. Let’s say you’re building a microservice. With three clicks in Backstage, you’ll have a new Spring Boot project with your repo automatically configured on GitHub and your CI already running the first build. + +### Golden Paths pave the way + +You can customize Backstage Software Templates to fit your organization’s standards. Using Go instead of Java? CircleCI instead of Jenkins? Serverless instead of Kubernetes? GCP instead of AWS? [Make your own recipes for any software component](https://backstage.io/docs/features/software-templates/adding-templates) and your best practices will be baked right in. + +## Getting started + +The sample Software Templates are available under `/create`. If you're setting up Backstage for the first time, follow [Getting Started with Backstage](https://backstage.io/docs/getting-started/) and go to `http://localhost:3000/create`. + +![available-templates](assets/2020-08-05/templates.png) + +### Step 1: Choose a template + +When you select a template that you want to create, you can ask for different input variables. These are then passed to the templater internally. + +![template-form](assets/2020-08-05/template-form.png) + +After filling in these variables, additional fields will appear so Backstage can be used. You’ll specify the owner, which is a `user` in the Backstage system, and the `Location`, which must be a GitHub organization and a non-existing GitHub repository name, formatted as `organization/reponame`. + +### Step 2: Run! + +Once you've entered values and confirmed, you'll then get a modal with live progress of what is currently happening with the creation of your template. + +![create-component](assets/2020-08-05/create-component.png) + +It shouldn't take too long before you see a success screen. At this point, a piece of “Hello World” software has been created in your repo, and the CI automatically picks it up and starts building the code. + +Your engineers don’t have to bother with setting up underlying infrastructure, it’s all built into the template. They can start focusing on delivering business value. + +### View new components in the Service Catalog + +New components, of course, get added automatically to the Backstage Service Catalog. After creation, you'll see the `View in Catalog` button, which will take you to the registered component in the catalog: + +![service-catalog](assets/2020-08-05/catalog.png) + +## Define your standards + +Backstage ships with four example templates, but since these are likely not the (only) ones you want to promote inside your company, the next step is to add [your own templates](https://backstage.io/docs/features/software-templates/software-templates-index). Using Backstage’s Software Templates feature, it’s easy to help your engineers get started building software with your organization’s best practices built-in. + +We have learned that one of the keys to getting these standards adopted is to keep an open process. Templates are code. By making it clear to your engineers that you are open to pull requests, and that teams with different needs can add their own templates, you are on the path of striking a good balance between autonomy and standardization. + +If you have feedback or questions, please open a [GitHub issue](https://github.com/spotify/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send me an email at [alund@spotify.com](mailto:alund@spotify.com) 🙏 + +To get regular product updates and news about the Backstage community, sign up for the [Backstage newsletter](https://mailchi.mp/spotify/backstage-community). diff --git a/microsite/blog/2020-09-08-announcing-tech-docs.md b/microsite/blog/2020-09-08-announcing-tech-docs.md new file mode 100644 index 0000000000..48b6eeb7cc --- /dev/null +++ b/microsite/blog/2020-09-08-announcing-tech-docs.md @@ -0,0 +1,120 @@ +--- +title: Announcing TechDocs: Spotify’s docs-like-code plugin for Backstage +author: Gary Niemen +authorURL: https://github.com/garyniemen +--- + +Since we [open sourced Backstage](https://backstage.io/blog/2020/03/16/announcing-backstage), one of the most requested features has been for a technical documentation plugin. Well, good news. The first open source version of TechDocs is here. Now let’s start collaborating and making it better, together. + + + + + +Internally, we call it TechDocs. It’s the most used plugin at Spotify by far — accounting for about 20% of our Backstage traffic (even though it is just one of 130+ plugins). Its popularity is evidence of something simple: We made documentation so easy to create, find, and use — people actually use it. + +We are quite sure the main reason for the success of TechDocs is our docs-like-code approach — engineers write their technical documentation in Markdown files that live together with the code. During CI, a documentation site is created using MkDocs, and all sites are rendered centrally in a Backstage plugin. On top of the static documentation, we incorporate additional metadata about the documentation site — such as owner, open GitHub Issues, Slack support channel, and Stack Overflow Enterprise tags. + +![available-templates](assets/announcing-techdocs/docs-in-backstage.png) + +But this is just one way to do it. Today we’re most excited for what the open version of TechDocs can become. + +## Okay, let’s start collaborating + +If you go to [GitHub](https://github.com/spotify/backstage/tree/master/plugins) now, you’ll find everything you need to start collaborating with us to build out the docs-like-code Backstage plugin — we’ll call it TechDocs in the open as well. + +You’ll find the code in [techdocs](https://github.com/spotify/backstage/tree/master/plugins/techdocs) (frontend) and [techdocs-backend](https://github.com/spotify/backstage/tree/master/plugins/techdocs-backend). (There are also two separate packages [techdocs-cli](https://github.com/spotify/backstage/tree/master/packages/techdocs-cli) and [techdocs-container](https://github.com/spotify/backstage/tree/master/packages/techdocs-container).) + +You’ll find issues to work on in the [issues queue](https://github.com/spotify/backstage/issues?q=is%3Aissue+is%3Aopen+label%3A%22docs-like-code%22+label%3A%22help+wanted%22), typically starting with TechDocs: and labeled with docs-like-code, some labeled good first issue. Feel free to add your own issues, of course. + +![available-templates](assets/announcing-techdocs/github-issues.png) + +What we have on GitHub so far is a first iteration of TechDocs that you can use end-to-end — in other words, from docs written in Markdown in GitHub to a published site on Backstage. + +More specifically, with this first iteration, you can: + +- Run TechDocs locally and read documentation. +- Configure your entity (e.g. service, website) so that Backstage builds your documentation and serves it in TechDocs. Documentation is displayed on the Docs tab in the Service Catalog and on its own page. +- Get documentation set up for free in your project when you create a new component out of one of the non-experimental templates (labeled with recommended). If you are looking for a standalone documentation project, use the docs-template. +- Choose your own storage solution for the documentation. +- Define your own API to interface with your documentation solution. + +For a full overview, including getting started instructions, check out our [TechDocs Documentation](https://backstage.io/docs/features/techdocs/techdocs-overview). + +But before you go there, let me tell you a bit about the TechDocs story — and why we believe TechDocs is such a powerful yet simple solution for great documentation. + +## The TechDocs story + +Here is the TechDocs story. It’s not an uncommon one (we have learned from many other companies). + +About a year and a half ago, we conducted a company-wide productivity survey. The third largest problem according to all our engineers? Not being able to find the technical information they needed to do their work. And it’s not surprising. There was no standard way to produce and consume technical documentation, so teams were going their own way — using Confluence, Google Docs, README files, custom built websites, GitHub Pages, and so on and on. And those searching for information were left to hunt for it in all those different places until they found what they were looking for (if they ever did). Worse, if you did happen to find the documentation that you needed, there was no way to know whether the information was up-to-date or correct. In other words, there was no way to know whether you could trust what you found. We did have technical writers at the company, but they were mostly scattered across the company solving documentation problems within their own particular domain. + +So this is the fertile soil on which TechDocs was built. + +After a Hack Week implementation attracted interest from high up in the company, we formed a cross-functional team made up of technical writers and engineers with the mission to solve internal technical documentation at Spotify. And we started to build TechDocs. We went for a docs-like-code approach, fiercely optimizing for engineers and engineering workflows. We also went for an opinionated approach, telling everybody: This is the standard way to do technical documentation at Spotify. The sense was that engineers appreciated a documentation solution that was in line with their workflow and, after all the frustration of multiple tools, were relieved to be told “this is the way to do it”. + +For more information about this journey, take a look at my 20-minute talk from DevRelCon London from last December: [The Hero’s Journey: How we are solving internal technical documentation at Spotify](https://www.linkedin.com/posts/garyniemen_how-we-are-solving-internal-technical-documentation-activity-6646078605594030080-4L31). + +## Key problem areas that we are solving + +We have come a long way, fast — both in our implementation and in our thinking. Here are some of the key problem areas that we are addressing. Note that they are in various stages of implementation, and we won’t be able to release everything within our minimum plugin. In fact, see this as an appetite taster. What we hope is that we can build together. + +### Stuck to unstuck + +Very early on, we decided that the main problem we were trying to solve was to help engineers (when using technical documentation) go from stuck to unstuck, and fast. This became our guiding principle. Is what we are building helping engineers get unstuck faster? From this, it follows that we need to promote quality documentation on the one hand, and provide a high level of discoverability on the other. One without the other is not going to cut it. + +### Feedback loops + +What we want to build is a thriving community of technical documentation creators, contributors, and readers. We want this because, we believe, this is the way to drive up the quality of the documentation. More readers, more feedback, more doc updates. And driving up the quality of the corpus of technical documentation leads to trust which in turn leads to more engagement and, hence, more of a thriving community. + +To get this working, we recognised that we need to remove ‘friction from the system’ — we need to build in efficient feedback loops. For example, help engineers get their doc site up by providing documentation improvement hints and build information as close as possible to where they are already working. And for readers, make it easy to give feedback. And then for doc site owners, ensure that they are notified when there is feedback and incentivised to make the fix. + +![available-templates](assets/announcing-techdocs/feedback-loop1.png) + +![available-templates](assets/announcing-techdocs/feedback-loop2.png) + +### Trust + +How do I know whether to trust this piece of documentation? This is a question we want to be able to answer for those using technical documentation in Backstage. It’s not an easy nut to crack. It is almost, one could say, the hard problem of technical documentation. For example, some might say ‘last updated’ is a key factor. But what about stable, good quality documentation that has no need to be updated? What about page views? Yes, this is a good sign that the documentation is being found and viewed, but it doesn’t say anything about whether the documentation can be trusted. How about a button: Did this documentation help? This is good, but will people use it? Will we get enough data to show trust? We have lofty ambitions of one day providing a trust score on the doc site informed by a super-intelligent algorithm. But we are not there yet. For now, we have landed on surfacing when the documentation was last updated, top five contributors, the support channel, owning team, and number of open GitHub Issues. But going forward we are definitely up for solving the hard problem. We think there’s much more work to be done here and look forward to seeing ideas from the community. + +### Discoverability and search + +How to find stuff? That is another big question. As mentioned above, it’s all well and good having quality documentation, but it’s no use whatsoever if you can’t find it. If you know what you are looking for, then you can use a search engine. If you don’t know what you are looking for, then you are going to need more — like a well designed information architecture, a user friendly browse experience, and even intelligent suggestions based on your role and what you have searched for previously. + +In this problem area, we made use of Elasticsearch, the open source search engine that was already being used in Backstage, to implement documentation search across all documentation sites and per documentation site. In terms of discoverability, we implemented a documentation home page in Backstage that surfaces Spotify’s most important documents and uses metrics to list the company’s most used doc sites as well as the documentation equivalent of a “your daily mix” playlist. + +![available-templates](assets/announcing-techdocs/discover1.png) + +![available-templates](assets/announcing-techdocs/discover2.png) + +There is much more to do in the area of discoverability and search. + +### Use case variations + +The standard use case for TechDocs is: One component in Backstage equals one GitHub repository, equals one doc site. This use case comes in two flavours: the repository is a code repository with docs or a docs-only repository. Then, to meet the needs of one large part of the Spotify engineering organisation that uses monorepos (multi-component repositories), we added a third use case. We built an MkDocs plugin that enabled doc site creators to include documentation from doc folders in other parts of the repository. So this use case is: One main component in Backstage equals a monorepo with distributed documentation, equals one doc site. + +These three use cases satisfy most of the needs, but we have had plenty of requests for additional use cases, for example, the ability to create multiple doc sites from a multi-component repository and the ability to create one doc site from documentation in multiple repositories. + +### Metrics + +There are many good arguments for standardizing the way that technical documentation is produced and consumed. One of them is metrics. If we have one way of producing technical documentation (in our case, GitHub Enterprise) and one place where it shows up (in our case, Backstage), we are in a strong position to build up metrics that help all the various stakeholders — for example, us building TechDocs, teams creating documentation sites, and engineers trying to get unstuck. Just imagine how much harder this would be if technical documentation was produced and consumed in a plethora of places, such as Confluence, Google Docs, README files, custom web sites, and GitHub Pages. + +One thing we have recently completed is a Manage page in Backstage for doc site owners. Here teams can see all the documentation that they own, the number of GitHub Issues per doc site or page, and last updated. We have also built a large dashboard using the open source analytics software Redash to inform our own product development process. + +![available-templates](assets/announcing-techdocs/metrics.png) + +Again, there is a lot more that can be done in the area of metrics. Did I mention the trust score? + +### Code-like-docs + +Code-like-docs, what? Okay, it’s just my little play on words. This is what I mean. One request that we keep getting is to be able to have code in the documentation fetched from and in sync with code in GitHub. In this way, you can avoid code in the documentation going stale. MkDocs does have an extension for this — but it has some limitations. For example, the code has to be in the /docs folder with the Markdown files. We are working on developing a wider and more flexible solution. + +### Golden Paths + +At Spotify, we have the concept of [Golden Paths](https://engineering.atspotify.com/2020/08/17/how-we-use-golden-paths-to-solve-fragmentation-in-our-software-ecosystem/) — one for each engineering discipline. My favourite definition of Golden Path is that it is the “opinionated and supported path”. Each Golden Path has an accompanying Golden Path tutorial that walks you through the opinionated and supported path. + +The Golden Path tutorials are Spotify’s most used and important documents and have shown themselves to be the most challenging to manage within a docs-like-code environment. One reason for this is that they are long, divided into many parts, and ownership is typically spread among many different teams. We have had to make use of GitHub codeowners to handle ownership and had to create datasets and data pipelines to be able to attach GitHub Issues to the specific parts or files that a team owns. Another challenge of the Golden Path tutorials is that parts are often dependent on other parts. We are just starting to look into how we can solve these dependency challenges in order to remove friction for engineers writing tutorial documentation. + +--- + +So that’s it for now. As you can see, we have come a long way AND there is much more to do. We are looking forward to continuing our docs-like-code journey out in the open with new, enthusiastic technical documentation friends. diff --git a/microsite/blog/assets/2/1.png b/microsite/blog/assets/2/1.png new file mode 100644 index 0000000000..21fb8bc26f Binary files /dev/null and b/microsite/blog/assets/2/1.png differ diff --git a/microsite/blog/assets/2/2.png b/microsite/blog/assets/2/2.png new file mode 100644 index 0000000000..16fa0cf5bf Binary files /dev/null and b/microsite/blog/assets/2/2.png differ diff --git a/microsite/blog/assets/2/3.png b/microsite/blog/assets/2/3.png new file mode 100644 index 0000000000..f610782186 Binary files /dev/null and b/microsite/blog/assets/2/3.png differ diff --git a/microsite/blog/assets/2/4.png b/microsite/blog/assets/2/4.png new file mode 100644 index 0000000000..d5e59ef92a Binary files /dev/null and b/microsite/blog/assets/2/4.png differ diff --git a/microsite/blog/assets/2/5.png b/microsite/blog/assets/2/5.png new file mode 100644 index 0000000000..00a6163c9a Binary files /dev/null and b/microsite/blog/assets/2/5.png differ diff --git a/microsite/blog/assets/2/screen.gif b/microsite/blog/assets/2/screen.gif new file mode 100644 index 0000000000..a0d52a23b4 Binary files /dev/null and b/microsite/blog/assets/2/screen.gif differ diff --git a/microsite/blog/assets/2/spotify-labs-header.png b/microsite/blog/assets/2/spotify-labs-header.png new file mode 100644 index 0000000000..8effd4b05a Binary files /dev/null and b/microsite/blog/assets/2/spotify-labs-header.png differ diff --git a/microsite/blog/assets/20-05-14/weaveworks-firekube-covid-19-spotify-backstage.png b/microsite/blog/assets/20-05-14/weaveworks-firekube-covid-19-spotify-backstage.png new file mode 100644 index 0000000000..fc5d05cd5d Binary files /dev/null and b/microsite/blog/assets/20-05-14/weaveworks-firekube-covid-19-spotify-backstage.png differ diff --git a/microsite/blog/assets/20-05-20/Service_Catalog_MVP.png b/microsite/blog/assets/20-05-20/Service_Catalog_MVP.png new file mode 100644 index 0000000000..4a50f17d15 Binary files /dev/null and b/microsite/blog/assets/20-05-20/Service_Catalog_MVP.png differ diff --git a/microsite/blog/assets/20-05-20/Service_Catalog_MVP_service.png b/microsite/blog/assets/20-05-20/Service_Catalog_MVP_service.png new file mode 100644 index 0000000000..8067f665c9 Binary files /dev/null and b/microsite/blog/assets/20-05-20/Service_Catalog_MVP_service.png differ diff --git a/microsite/blog/assets/20-05-20/tabs.png b/microsite/blog/assets/20-05-20/tabs.png new file mode 100644 index 0000000000..4a5b11f6fe Binary files /dev/null and b/microsite/blog/assets/20-05-20/tabs.png differ diff --git a/microsite/blog/assets/20-07-01/auth-landing.png b/microsite/blog/assets/20-07-01/auth-landing.png new file mode 100644 index 0000000000..e00f4c2e43 Binary files /dev/null and b/microsite/blog/assets/20-07-01/auth-landing.png differ diff --git a/microsite/blog/assets/20-07-01/auth-sidebar.png b/microsite/blog/assets/20-07-01/auth-sidebar.png new file mode 100644 index 0000000000..b089ab6c0a Binary files /dev/null and b/microsite/blog/assets/20-07-01/auth-sidebar.png differ diff --git a/microsite/blog/assets/2020-08-05/cards.png b/microsite/blog/assets/2020-08-05/cards.png new file mode 100644 index 0000000000..4779618b92 Binary files /dev/null and b/microsite/blog/assets/2020-08-05/cards.png differ diff --git a/microsite/blog/assets/2020-08-05/catalog.png b/microsite/blog/assets/2020-08-05/catalog.png new file mode 100644 index 0000000000..e9c0c65ade Binary files /dev/null and b/microsite/blog/assets/2020-08-05/catalog.png differ diff --git a/microsite/blog/assets/2020-08-05/create-component.png b/microsite/blog/assets/2020-08-05/create-component.png new file mode 100644 index 0000000000..4d815393fc Binary files /dev/null and b/microsite/blog/assets/2020-08-05/create-component.png differ diff --git a/microsite/blog/assets/2020-08-05/feature.mp4 b/microsite/blog/assets/2020-08-05/feature.mp4 new file mode 100644 index 0000000000..a42d9da0e6 Binary files /dev/null and b/microsite/blog/assets/2020-08-05/feature.mp4 differ diff --git a/microsite/blog/assets/2020-08-05/template-form.png b/microsite/blog/assets/2020-08-05/template-form.png new file mode 100644 index 0000000000..5805243f59 Binary files /dev/null and b/microsite/blog/assets/2020-08-05/template-form.png differ diff --git a/microsite/blog/assets/2020-08-05/templates.png b/microsite/blog/assets/2020-08-05/templates.png new file mode 100644 index 0000000000..e350d463f6 Binary files /dev/null and b/microsite/blog/assets/2020-08-05/templates.png differ diff --git a/microsite/blog/assets/3/audit-list.png b/microsite/blog/assets/3/audit-list.png new file mode 100644 index 0000000000..84b64f976a Binary files /dev/null and b/microsite/blog/assets/3/audit-list.png differ diff --git a/microsite/blog/assets/3/audit-view.png b/microsite/blog/assets/3/audit-view.png new file mode 100644 index 0000000000..a2e9716cf4 Binary files /dev/null and b/microsite/blog/assets/3/audit-view.png differ diff --git a/microsite/blog/assets/3/create-audit.png b/microsite/blog/assets/3/create-audit.png new file mode 100644 index 0000000000..cb28de73a4 Binary files /dev/null and b/microsite/blog/assets/3/create-audit.png differ diff --git a/microsite/blog/assets/3/lead-copy.png b/microsite/blog/assets/3/lead-copy.png new file mode 100644 index 0000000000..fbf247eec8 Binary files /dev/null and b/microsite/blog/assets/3/lead-copy.png differ diff --git a/microsite/blog/assets/3/lead.png b/microsite/blog/assets/3/lead.png new file mode 100644 index 0000000000..4b60c4961c Binary files /dev/null and b/microsite/blog/assets/3/lead.png differ diff --git a/docs/getting-started/create-app_output.png b/microsite/blog/assets/4/create-app.png similarity index 100% rename from docs/getting-started/create-app_output.png rename to microsite/blog/assets/4/create-app.png diff --git a/microsite/blog/assets/4/welcome.png b/microsite/blog/assets/4/welcome.png new file mode 100644 index 0000000000..5de0d57098 Binary files /dev/null and b/microsite/blog/assets/4/welcome.png differ diff --git a/microsite/blog/assets/5/lead.png b/microsite/blog/assets/5/lead.png new file mode 100644 index 0000000000..657268fc09 Binary files /dev/null and b/microsite/blog/assets/5/lead.png differ diff --git a/microsite/blog/assets/6/header.png b/microsite/blog/assets/6/header.png new file mode 100644 index 0000000000..6908e40dbc Binary files /dev/null and b/microsite/blog/assets/6/header.png differ diff --git a/microsite/blog/assets/announcing-techdocs/discover1.png b/microsite/blog/assets/announcing-techdocs/discover1.png new file mode 100644 index 0000000000..5b23e64b43 Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/discover1.png differ diff --git a/microsite/blog/assets/announcing-techdocs/discover2.png b/microsite/blog/assets/announcing-techdocs/discover2.png new file mode 100644 index 0000000000..3737ee68db Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/discover2.png differ diff --git a/microsite/blog/assets/announcing-techdocs/docs-in-backstage.png b/microsite/blog/assets/announcing-techdocs/docs-in-backstage.png new file mode 100644 index 0000000000..f3b724ff4f Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/docs-in-backstage.png differ diff --git a/microsite/blog/assets/announcing-techdocs/feedback-loop1.png b/microsite/blog/assets/announcing-techdocs/feedback-loop1.png new file mode 100644 index 0000000000..4e2d0de5df Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/feedback-loop1.png differ diff --git a/microsite/blog/assets/announcing-techdocs/feedback-loop2.png b/microsite/blog/assets/announcing-techdocs/feedback-loop2.png new file mode 100644 index 0000000000..516780e3ac Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/feedback-loop2.png differ diff --git a/microsite/blog/assets/announcing-techdocs/github-issues.png b/microsite/blog/assets/announcing-techdocs/github-issues.png new file mode 100644 index 0000000000..3d5f3465c0 Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/github-issues.png differ diff --git a/microsite/blog/assets/announcing-techdocs/metrics.png b/microsite/blog/assets/announcing-techdocs/metrics.png new file mode 100644 index 0000000000..b332a65b0d Binary files /dev/null and b/microsite/blog/assets/announcing-techdocs/metrics.png differ diff --git a/microsite/blog/assets/blog_1.png b/microsite/blog/assets/blog_1.png new file mode 100644 index 0000000000..f8c3516fa7 Binary files /dev/null and b/microsite/blog/assets/blog_1.png differ diff --git a/microsite/blog/assets/illustration.svg b/microsite/blog/assets/illustration.svg new file mode 100644 index 0000000000..50e865ed4f --- /dev/null +++ b/microsite/blog/assets/illustration.svg @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/microsite/core/Components.js b/microsite/core/Components.js new file mode 100644 index 0000000000..49e7d7daca --- /dev/null +++ b/microsite/core/Components.js @@ -0,0 +1,114 @@ +const React = require('react'); +const PropTypes = require('prop-types'); +const simpleComponent = (Component, baseClassName = '', mods = []) => { + const SimpleComponent = props => { + // Extra BEM modifiers, e.g. `Block__Container--reversed` + const modClasses = []; + const otherProps = {}; + for (const prop in props) { + if (mods.indexOf(prop) !== -1) { + modClasses.push(`${baseClassName}--${prop}`); + } else { + otherProps[prop] = props[prop]; + } + } + + return ( + + ); + }; + SimpleComponent.displayName = `SimpleComponent(${Component}, ${baseClassName})`; + + SimpleComponent.propTypes = {}; + for (const mod of mods) { + SimpleComponent.propTypes[mod] = PropTypes.bool; + } + + return SimpleComponent; +}; + +const Block = simpleComponent('section', 'Block', ['small', 'wrapped']); +Block.Container = simpleComponent('div', 'Block__Container', [ + 'reversed', + 'wrapped', + 'column', +]); +Block.TitleBox = simpleComponent('h1', 'Block__TitleBox', ['large', 'story']); +Block.TextBox = simpleComponent('div', 'Block__TextBox', ['wide', 'small']); + +Block.Title = simpleComponent('h1', 'Block__Title', ['half', 'main']); +Block.Subtitle = simpleComponent('h1', 'Block__Subtitle'); + +Block.SmallTitle = simpleComponent('h2', 'Block__SmallTitle'); +Block.SmallestTitle = simpleComponent('h3', 'Block__SmallestTitle'); + +const BulletLine = simpleComponent('div', 'BulletLine'); + +Block.Paragraph = simpleComponent('p', 'Block__Paragraph'); +Block.LinkButton = simpleComponent('a', 'Block__LinkButton', ['stretch']); +Block.QuoteContainer = simpleComponent('div', 'Block__QuoteContainer'); +Block.Quote = simpleComponent('p', 'Block__Quote'); +Block.Divider = simpleComponent('p', 'Block__Divider', ['quote']); +Block.MediaFrame = simpleComponent('div', 'Block__MediaFrame'); +Block.Graphics = ({ padding, children }) => { + const style = {}; + if (padding) { + style.padding = `${padding}% 0`; + } + return ( +
+
+
+ ); +}; +Block.Graphic = props => { + /* Coordinates and size are in % of graphics container size, e.g. width={50} is 50% of parent width */ + const { x = 0, y = 0, width = 0, src, className = '' } = props; + const style = Object.assign( + { left: `${x}%`, top: `${y}%`, width: `${width}%` }, + props.style, + ); + return ( + + ); +}; + +Block.Image = props => { + /* Coordinates and size are in % of graphics container size, e.g. width={50} is 50% of parent width */ + return ( +
+ ); +}; + +const ActionBlock = simpleComponent('section', 'ActionBlock'); +ActionBlock.Title = simpleComponent('h1', 'ActionBlock__Title'); +ActionBlock.Subtitle = simpleComponent('h2', 'ActionBlock__Subtitle'); +ActionBlock.Link = simpleComponent('a', 'ActionBlock__Link'); + +const Breakpoint = ({ narrow, wide }) => ( + +
{narrow}
+
{wide}
+ +); + +module.exports = { + Block, + ActionBlock, + Breakpoint, + BulletLine, +}; diff --git a/microsite/core/Footer.js b/microsite/core/Footer.js new file mode 100644 index 0000000000..761862c1fc --- /dev/null +++ b/microsite/core/Footer.js @@ -0,0 +1,99 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +const React = require('react'); + +class Footer extends React.Component { + docUrl(doc, language) { + const baseUrl = this.props.config.baseUrl; + const docsUrl = this.props.config.docsUrl; + const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; + const langPart = `${language ? `${language}/` : ''}`; + return `${baseUrl}${docsPart}${langPart}${doc}`; + } + + pageUrl(doc, language) { + const baseUrl = this.props.config.baseUrl; + return baseUrl + (language ? `${language}/` : '') + doc; + } + + render() { + return ( + + ); + } +} + +module.exports = Footer; diff --git a/microsite/core/GridBlockWithButton.js b/microsite/core/GridBlockWithButton.js new file mode 100644 index 0000000000..0f77a6a148 --- /dev/null +++ b/microsite/core/GridBlockWithButton.js @@ -0,0 +1,100 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +const React = require('react'); +const classNames = require('classnames'); + +const CompLibrary = require(`${process.cwd()}/node_modules/docusaurus/lib/core/CompLibrary.js`); +const MarkdownBlock = CompLibrary.MarkdownBlock; /* Used to read markdown */ + +class GridBlockWithButton extends React.Component { + renderBlock(origBlock) { + const blockDefaults = { + imageAlign: 'left', + }; + + const block = { + ...blockDefaults, + ...origBlock, + }; + + const blockClasses = classNames('blockElement', this.props.className, { + alignCenter: this.props.align === 'center', + alignRight: this.props.align === 'right', + fourByGridBlock: this.props.layout === 'fourColumn', + imageAlignSide: + block.image && + (block.imageAlign === 'left' || block.imageAlign === 'right'), + imageAlignTop: block.image && block.imageAlign === 'top', + imageAlignRight: block.image && block.imageAlign === 'right', + imageAlignBottom: block.image && block.imageAlign === 'bottom', + imageAlignLeft: block.image && block.imageAlign === 'left', + threeByGridBlock: this.props.layout === 'threeColumn', + twoByGridBlock: this.props.layout === 'twoColumn', + }); + + const topLeftImage = + (block.imageAlign === 'top' || block.imageAlign === 'left') && + this.renderBlockImage(block.image, block.imageLink, block.imageAlt); + + const bottomRightImage = + (block.imageAlign === 'bottom' || block.imageAlign === 'right') && + this.renderBlockImage(block.image, block.imageLink, block.imageAlt); + + return ( +
+ {topLeftImage} +
+ {this.renderBlockTitle(block.title)} + {block.content} + +
+ {bottomRightImage} +
+ ); + } + + renderBlockImage(image) { + if (!image) { + return null; + } + + return
{image}
; + } + + renderBlockTitle(title) { + if (!title) { + return null; + } + + return ( +

+ {title} +

+ ); + } + + render() { + return ( +
+ {this.props.contents.map(this.renderBlock, this)} +
+ ); + } +} + +GridBlockWithButton.defaultProps = { + align: 'left', + contents: [], + layout: 'twoColumn', +}; + +module.exports = GridBlockWithButton; diff --git a/microsite/data/plugins/api-docs.yaml b/microsite/data/plugins/api-docs.yaml new file mode 100644 index 0000000000..babcbf74f1 --- /dev/null +++ b/microsite/data/plugins/api-docs.yaml @@ -0,0 +1,9 @@ +--- +title: API Docs +author: SDA SE +authorUrl: https://sda.se/ +category: Discovery +description: Components to discover and display API entities as an extension to the catalog plugin. +documentation: https://github.com/spotify/backstage/blob/master/plugins/api-docs/README.md +iconUrl: https://thecoders.io/wp-content/uploads/2019/11/tech-swagger.svg +npmPackageName: '@backstage/plugin-api-docs' diff --git a/microsite/data/plugins/circleci.yaml b/microsite/data/plugins/circleci.yaml new file mode 100644 index 0000000000..5f7b4b08e2 --- /dev/null +++ b/microsite/data/plugins/circleci.yaml @@ -0,0 +1,12 @@ +--- +title: CircleCI +author: Spotify +authorUrl: https://github.com/spotify +category: CI +description: Automate your development process with CI hosted in the cloud or on a private server. +documentation: https://github.com/spotify/backstage/tree/master/plugins/circleci +iconUrl: https://www.saaves.com/storage/brochure/logo-circleci-icon1583764538.png +npmPackageName: '@backstage/plugin-circleci' +tags: + - ci + - cd diff --git a/microsite/data/plugins/github-actions.yaml b/microsite/data/plugins/github-actions.yaml new file mode 100644 index 0000000000..062d0fff61 --- /dev/null +++ b/microsite/data/plugins/github-actions.yaml @@ -0,0 +1,13 @@ +--- +title: GitHub Actions +author: Spotify +authorUrl: https://github.com/spotify +category: CI +description: GitHub Actions makes it easy to automate all your software workflows, now with world-class CI/CD. Build, test, and deploy your code right from GitHub. +documentation: https://github.com/spotify/backstage/tree/master/plugins/github-actions +iconUrl: https://avatars2.githubusercontent.com/u/44036562?s=400&v=4 +npmPackageName: '@backstage/plugin-github-actions' +tags: + - ci + - cd + - github diff --git a/microsite/data/plugins/github-pull-requests.yaml b/microsite/data/plugins/github-pull-requests.yaml new file mode 100644 index 0000000000..6479452c8e --- /dev/null +++ b/microsite/data/plugins/github-pull-requests.yaml @@ -0,0 +1,9 @@ +--- +title: GitHub Pull Requests +author: roadie.io +authorUrl: https://roadie.io/ +category: CI +description: View GitHub pull requests for your service in Backstage. +documentation: https://roadie.io/backstage/plugins/github-pull-requests +iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png +npmPackageName: '@roadiehq/backstage-plugin-github-pull-requests' diff --git a/microsite/data/plugins/gitops-cluster.yaml b/microsite/data/plugins/gitops-cluster.yaml new file mode 100644 index 0000000000..6f8ab6b097 --- /dev/null +++ b/microsite/data/plugins/gitops-cluster.yaml @@ -0,0 +1,14 @@ +--- +title: GitOps Clusters +author: Weaveworks +authorUrl: https://www.weave.works/ +category: Kubernetes +description: Create GitOps-managed Kubernetes clusters. Currently, it supports provisioning EKS clusters on GitHub via GitHub Actions. +documentation: https://github.com/spotify/backstage/tree/master/plugins/gitops-profiles +iconUrl: https://res-5.cloudinary.com/crunchbase-production/image/upload/c_lpad,h_256,w_256,f_auto,q_auto:eco/v1462316670/i9d3delzvx1erzjhmcws.png +npmPackageName: '@backstage/plugin-gitops-profiles' +tags: + - kubernetes + - gitops + - github + - eks diff --git a/microsite/data/plugins/graphiql.yaml b/microsite/data/plugins/graphiql.yaml new file mode 100644 index 0000000000..e997360b04 --- /dev/null +++ b/microsite/data/plugins/graphiql.yaml @@ -0,0 +1,12 @@ +--- +title: GraphiQL +author: Spotify +authorUrl: https://github.com/spotify +category: Debugging +description: Integrates GraphiQL as a tool to browse GraphQL API endpoints inside Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/graphiql +iconUrl: https://upload.wikimedia.org/wikipedia/commons/thumb/1/17/GraphQL_Logo.svg/1024px-GraphQL_Logo.svg.png +npmPackageName: '@backstage/plugin-graphiql' +tags: + - graphql + - graphiql diff --git a/microsite/data/plugins/jenkins.yaml b/microsite/data/plugins/jenkins.yaml new file mode 100644 index 0000000000..78b3616595 --- /dev/null +++ b/microsite/data/plugins/jenkins.yaml @@ -0,0 +1,12 @@ +--- +title: Jenkins +author: '@timja' +authorUrl: https://github.com/timja +category: CI +description: Jenkins offers a simple way to set up a continuous integration and continuous delivery environment. +documentation: https://github.com/spotify/backstage/tree/master/plugins/jenkins +iconUrl: https://img.icons8.com/color/1600/jenkins.png +npmPackageName: '@backstage/plugin-jenkins' +tags: + - ci + - cd diff --git a/microsite/data/plugins/lighthouse.yaml b/microsite/data/plugins/lighthouse.yaml new file mode 100644 index 0000000000..0819455ed2 --- /dev/null +++ b/microsite/data/plugins/lighthouse.yaml @@ -0,0 +1,14 @@ +--- +title: Lighthouse +author: Spotify +authorUrl: https://github.com/spotify +category: Accessibility +description: Google's Lighthouse tool is a great resource for benchmarking and improving the accessibility, performance, SEO, and best practices of your website. +documentation: https://github.com/spotify/backstage/tree/master/plugins/lighthouse +iconUrl: https://seeklogo.com/images/G/google-lighthouse-logo-1C7FA08580-seeklogo.com.png +npmPackageName: '@backstage/plugin-lighthouse' +tags: + - web + - seo + - accessibility + - performance diff --git a/microsite/data/plugins/new-relic.yaml b/microsite/data/plugins/new-relic.yaml new file mode 100644 index 0000000000..e3ddf18652 --- /dev/null +++ b/microsite/data/plugins/new-relic.yaml @@ -0,0 +1,14 @@ +--- +title: New Relic +author: '@timwheelercom' +authorUrl: https://github.com/timwheelercom +category: Monitoring +description: Observability platform built to help engineers create and monitor their software. +documentation: https://github.com/spotify/backstage/tree/master/plugins/newrelic +iconUrl: https://www.mulesoft.com/sites/default/files/2018-10/New_relic.png +npmPackageName: '@backstage/plugin-newrelic' +tags: + - performance + - monitoring + - errors + - alerting diff --git a/microsite/data/plugins/rollbar.yaml b/microsite/data/plugins/rollbar.yaml new file mode 100644 index 0000000000..8ef3d8d66a --- /dev/null +++ b/microsite/data/plugins/rollbar.yaml @@ -0,0 +1,9 @@ +--- +title: Rollbar +author: '@andrewthauer' +authorUrl: https://github.com/andrewthauer +category: Monitoring +description: View Rollbar errors for your services in Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/rollbar +iconUrl: https://rollbar.com/assets/media/rollbar-mark-color.png +npmPackageName: '@backstage/plugin-rollbar' diff --git a/microsite/data/plugins/sentry.yaml b/microsite/data/plugins/sentry.yaml new file mode 100644 index 0000000000..8da488f727 --- /dev/null +++ b/microsite/data/plugins/sentry.yaml @@ -0,0 +1,9 @@ +--- +title: Sentry +author: Spotify +authorUrl: https://github.com/spotify +category: Monitoring +description: View Sentry issues in Backstage. +documentation: https://github.com/spotify/backstage/tree/master/plugins/sentry +iconUrl: https://sentry-brand.storage.googleapis.com/sentry-glyph-white.png +npmPackageName: '@backstage/plugin-sentry' diff --git a/microsite/data/plugins/tech-radar.yaml b/microsite/data/plugins/tech-radar.yaml new file mode 100644 index 0000000000..b1d4f5cc17 --- /dev/null +++ b/microsite/data/plugins/tech-radar.yaml @@ -0,0 +1,9 @@ +--- +title: Tech Radar +author: Spotify +authorUrl: https://github.com/spotify +category: Discovery +description: Visualize the your company's official guidelines of different areas of software development. +documentation: https://github.com/spotify/backstage/tree/master/plugins/tech-radar +iconUrl: https://www.materialui.co/materialIcons/action/track_changes_white_192x192.png +npmPackageName: '@backstage/plugin-tech-radar' diff --git a/microsite/data/plugins/travis-ci.yaml b/microsite/data/plugins/travis-ci.yaml new file mode 100644 index 0000000000..520b884c20 --- /dev/null +++ b/microsite/data/plugins/travis-ci.yaml @@ -0,0 +1,9 @@ +--- +title: Travis CI +author: roadie.io +authorUrl: https://roadie.io/ +category: CI +description: View Travis CI builds for your service in Backstage. +documentation: https://roadie.io/backstage/plugins/travis-ci +iconUrl: https://roadie.io/static/af2941eaf0af675facb281d566f42e14/45f2b/travis-ci-mascot-200x200.png +npmPackageName: '@roadiehq/backstage-plugin-travis-ci' diff --git a/microsite/package.json b/microsite/package.json new file mode 100644 index 0000000000..734f1e4d49 --- /dev/null +++ b/microsite/package.json @@ -0,0 +1,19 @@ +{ + "version": "0.0.0", + "name": "backstage-microsite", + "license": "Apache-2.0", + "private": true, + "scripts": { + "examples": "docusaurus-examples", + "start": "docusaurus-start", + "build": "docusaurus-build", + "publish-gh-pages": "docusaurus-publish", + "write-translations": "docusaurus-write-translations", + "version": "docusaurus-version", + "rename-version": "docusaurus-rename-version" + }, + "devDependencies": { + "docusaurus": "^2.0.0-alpha.64", + "js-yaml": "^3.14.0" + } +} diff --git a/microsite/pages/en/demos.js b/microsite/pages/en/demos.js new file mode 100644 index 0000000000..d891e05613 --- /dev/null +++ b/microsite/pages/en/demos.js @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +const React = require('react'); +const Components = require(`${process.cwd()}/core/Components.js`); +const Block = Components.Block; + +const Background = props => { + const { config: siteConfig } = props; + const { baseUrl } = siteConfig; + return ( +
+ + + + See us in action. + + To illustrate the potential of Backstage, we’re showing you{' '} + + how we use it here at Spotify + + . The videos below feature the custom plugins and templates that + we built for our internal version of Backstage. You can use + Backstage to build the developer portal you want — integrating the + tools that you use in your own infrastructure ecosystem. (Or get + started by building an open source plugin for Backstage that + everyone can use, like our{' '} + + Lighthouse Plugin + + .) + + + + + + + + + + + + Introduction to Backstage + + Backstage is an open source platform for building developer + portals. We’ve been using our homegrown version at Spotify for + years — so it’s already packed with features. (We have over 120 + internal plugins, built by 60 different teams.) In this live demo + recording, Stefan Ålund, product manager for Backstage, tells the + origin story of Backstage and gives you a tour of how we use it + here at Spotify. + + + Watch now + + + + + + + + + + + + + Make documentation easy + + + Documentation! Everyone needs it, no one wants to create it, and + no one can ever find it. Backstage follows a “docs like code” + approach: you write documentation in Markdown files right + alongside your code. This makes documentation easier to create, + maintain, find — and, you know, actually use. This demo video + showcases Spotify’s internal version of TechDocs. Learn more about{' '} + + TechDocs + + . + + + Watch now + + + + + + + + + + + + Manage your tech health + + Instead of manually updating a spreadsheet, what if you had a + beautiful dashboard that could give you an instant, interactive + picture of your entire org’s tech stack? That’s how we do it at + Spotify. With our Tech Insights plugin for Backstage, anyone at + Spotify can see which version of which software anyone else at + Spotify is using — and a whole a lot more. From managing + migrations to fighting tech entropy, Backstage makes managing our + tech health actually kind of pleasant. + + + + Watch now + + + + + + + + + + + + Create a microservice + + You’re a Spotify engineer about to build a new microservice (or + any component) using Spring Boot. Where do you start? Search for a + quick start guide online? Create an empty repo on GitHub? Copy and + paste an old project? Nope. Just go to Backstage, and you’ll be up + and running in two minutes — with a “Hello World” app, CI, and + documentation all automatically set up and configured in a + standardized way. + + + + Watch now + + + + + + + + + + + + Search all your services + + All of Spotify’s services are automatically indexed in Backstage. + So our engineers can stop playing detective — no more spamming + Slack channels asking if anyone knows who owns a particular + service and where you can find its API, only to discover that the + owner went on sabbatical three months ago and you have to hunt + them down on a mountain in Tibet where they’re on a 12-day silent + meditation retreat. At Spotify, anyone can always find anyone + else’s service, inspect its APIs, and contact its current owner — + all with one search. + + + Watch now + + + + + + + + + + + + Manage data pipelines + + We manage a lot of data pipelines (also known as workflows) here + at Spotify. So, of course, we made a great workflows plugin for + our version of Backstage. All our workflow tools — including a + scheduler, log inspector, data lineage graph, and configurable + alerts — are integrated into one simple interface. + + + Watch now + + + + + + + +
+ ); +}; + +module.exports = Background; diff --git a/microsite/pages/en/docs.js b/microsite/pages/en/docs.js new file mode 100644 index 0000000000..a8bda18f88 --- /dev/null +++ b/microsite/pages/en/docs.js @@ -0,0 +1,12 @@ +const React = require('react'); +const Redirect = require('../../core/Redirect.js'); + +const siteConfig = require(process.cwd() + '/siteConfig.js'); + +function Docs() { + return ( + + ); +} + +module.exports = Docs; diff --git a/microsite/pages/en/index.js b/microsite/pages/en/index.js new file mode 100644 index 0000000000..37298ae8e2 --- /dev/null +++ b/microsite/pages/en/index.js @@ -0,0 +1,483 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +const React = require('react'); +const Components = require(`${process.cwd()}/core/Components.js`); +const Block = Components.Block; +const ActionBlock = Components.ActionBlock; +const Breakpoint = Components.Breakpoint; +const BulletLine = Components.BulletLine; + +class Index extends React.Component { + render() { + const { config: siteConfig } = this.props; + const { baseUrl } = siteConfig; + + return ( +
+ + + + + An open platform for building developer portals + + + Powered by a centralized service catalog, Backstage restores + order to your infrastructure. So your product teams can ship + high-quality code quickly — without compromising autonomy. + + + GitHub + + + + + + + + + + + + + + The Speed Paradox + + At Spotify, we’ve always believed in the speed and ingenuity + that comes from having autonomous development teams. But as we + learned firsthand, the faster you grow, the more fragmented and + complex your software ecosystem becomes. And then everything + slows down again. + + + + + The Standards Paradox + + By centralizing services and standardizing your tooling, + Backstage streamlines your development environment from end to + end. Instead of restricting autonomy, standardization frees your + engineers from infrastructure complexity. So you can return to + building and scaling, quickly and safely. + + + + + + + + + {' '} + + + Backstage Service Catalog{' '} + + (alpha) + + + + Build an ecosystem, not a wilderness + + + + + + } + /> + + + + Manage all your software, all in one place{' '} + + + Backstage makes it easy for one team to manage 10 services — and + makes it possible for your company to manage thousands of them + + + + + A uniform overview + + Every team can see all the services they own and related + resources (deployments, data pipelines, pull request status, + etc.) + + + + + + Metadata on tap + + All that information can be shared with plugins inside Backstage + to enable other management features, like resource monitoring + and testing + + + + + + Not just services + + Libraries, websites, ML models — you name it, Backstage knows + all about it, including who owns it, dependencies, and more + + + + + + + Discoverability & accountability + + + No more orphan software hiding in the dark corners of your tech + stack + + + + + + + + + Learn more about the service catalog + + + Read + + + + + + + + + Backstage Software Templates{' '} + + (alpha) + + + Standards can set you free + + + + + } + /> + + + + Like automated getting started guides + + + Using templates, engineers can spin up a new microservice with + your organization’s best practices built-in, right from the + start + + + + + + Push-button deployment + + Click a button to create a Spring Boot project with your repo + automatically configured on GitHub and your CI already running + the first build + + + + + + Built to your standards + + Go instead of Java? CircleCI instead of Jenkins? Serverless + instead of Kubernetes? GCP instead of AWS? Customize your + recipes with your best practices baked-in + + + + + + + Golden Paths pave the way + + + When the right way is also the easiest way, engineers get up and + running faster — and more safely + + + + + + } + /> + + + + + + Build your own software templates + + + Contribute + + + + + + + + + Backstage TechDocs + Docs like code + + + + + + + } + /> + + + Free documentation + + Whenever you use a Backstage Software Template, your project + automatically gets a TechDocs site, for free + + + + + + Easy to write + + With our docs-like-code approach, engineers write their + documentation in Markdown files right alongside their code + + + + + + Easy to maintain + + Updating code? Update your documentation while you’re there — + with docs and code in the same place, it becomes a natural part + of your workstream + + + + + + Easy to find and use + + Since all your documentation is in Backstage, finding any + TechDoc is just a search query away + + + + + + + } + /> + + + + + Learn more about TechDocs + + Docs + + + + + + + + + Customize Backstage with plugins + + An app store for your infrastructure + + + + + + } + /> + + + Add functionality + + Want scalable website testing? Add the{' '} + + Lighthouse + {' '} + plugin. Wondering about recommended frameworks? Add the{' '} + + Tech Radar + {' '} + plugin.{' '} + + + + + + BYO Plugins + + If you don’t see the plugin you need, it’s simple to build your + own + + + + + + + Integrate your own custom tooling + + + Building internal plugins lets you tailor your version of + Backstage to be a perfect fit for your infrastructure + + + + + + + Share with the community + + + Building open source plugins contributes + to the entire Backstage ecosystem, which benefits everyone + + + + } + /> + + + + + Build a plugin + + Contribute + + + + + + + Backstage is a{' '} + + Cloud Native Computing Foundation + {' '} + sandbox project + +
+ + +
+ ); + } +} + +module.exports = Index; diff --git a/microsite/pages/en/plugins.js b/microsite/pages/en/plugins.js new file mode 100644 index 0000000000..0ef8791970 --- /dev/null +++ b/microsite/pages/en/plugins.js @@ -0,0 +1,119 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +const fs = require('fs'); +const yaml = require('js-yaml'); +const React = require('react'); +const Components = require(`${process.cwd()}/core/Components.js`); +const { + Block: { Container }, + BulletLine, +} = Components; + +const pluginsDirectory = require('path').join(process.cwd(), 'data/plugins'); +const pluginMetadata = fs + .readdirSync(pluginsDirectory) + .map(file => + yaml.safeLoad(fs.readFileSync(`./data/plugins/${file}`, 'utf8')), + ); +const truncate = text => + text.length > 170 ? text.substr(0, 170) + '...' : text; + +const addPluginDocsLink = '/docs/plugins/add-to-marketplace'; +const defaultIconUrl = 'img/logo-gradient-on-dark.svg'; + +const Plugins = () => ( +
+
+
+

Plugin marketplace

+

+ Open source plugins that you can add to your Backstage deployment. + Learn how to build a plugin. +

+ + + Add to marketplace + + +
+ + + {pluginMetadata.map( + ({ + iconUrl, + title, + description, + author, + authorUrl, + documentation, + category, + }) => ( +
+
+ {title} +

{title}

+

+ by {author} +

+ {category} +
+
+

{truncate(description)}

+
+ + + + Explore + + + +
+ ), + )} +
+
+

+ Do you have an existing plugin that you want to add to the + Marketplace? +

+

+ + Add to marketplace + +

+
+ +

+ See what plugins are already{' '} + + in progress + {' '} + and 👍. Missing a plugin for your favorite tool? Please{' '} + + suggest + {' '} + a new one. +

+ +
+ +
+
+); + +module.exports = Plugins; diff --git a/microsite/sidebars.json b/microsite/sidebars.json new file mode 100644 index 0000000000..8fa4935a38 --- /dev/null +++ b/microsite/sidebars.json @@ -0,0 +1,167 @@ +{ + "docs": { + "Overview": [ + "overview/what-is-backstage", + "overview/architecture-overview", + "overview/architecture-terminology", + "overview/roadmap", + "overview/vision", + "overview/background", + "overview/adopting", + "overview/logos" + ], + "Getting Started": [ + "getting-started/index", + "getting-started/running-backstage-locally", + "getting-started/installation", + "getting-started/development-environment", + "getting-started/create-an-app", + { + "type": "subcategory", + "label": "App configuration", + "ids": [ + "getting-started/configure-app-with-plugins", + "getting-started/app-custom-theme" + ] + }, + { + "type": "subcategory", + "label": "Deployment", + "ids": [ + "getting-started/deployment-k8s", + "getting-started/deployment-other" + ] + } + ], + "Core Features": [ + { + "type": "subcategory", + "label": "Software Catalog", + "ids": [ + "features/software-catalog/software-catalog-overview", + "features/software-catalog/installation", + "features/software-catalog/configuration", + "features/software-catalog/system-model", + "features/software-catalog/descriptor-format", + "features/software-catalog/well-known-annotations", + "features/software-catalog/extending-the-model", + "features/software-catalog/external-integrations", + "features/software-catalog/software-catalog-api" + ] + }, + { + "type": "subcategory", + "label": "Software Templates", + "ids": [ + "features/software-templates/software-templates-index", + "features/software-templates/installation", + "features/software-templates/adding-templates", + "features/software-templates/extending/extending-index", + "features/software-templates/extending/extending-templater", + "features/software-templates/extending/extending-publisher", + "features/software-templates/extending/extending-preparer" + ] + }, + { + "type": "subcategory", + "label": "TechDocs", + "ids": [ + "features/techdocs/techdocs-overview", + "features/techdocs/getting-started", + "features/techdocs/concepts", + "features/techdocs/architecture", + "features/techdocs/creating-and-publishing", + "features/techdocs/faqs" + ] + } + ], + "Plugins": [ + "plugins/index", + "plugins/existing-plugins", + "plugins/create-a-plugin", + "plugins/plugin-development", + "plugins/structure-of-a-plugin", + "plugins/integrating-plugin-into-service-catalog", + { + "type": "subcategory", + "label": "Backends and APIs", + "ids": [ + "plugins/proxying", + "plugins/backend-plugin", + "plugins/call-existing-api" + ] + }, + { + "type": "subcategory", + "label": "Testing", + "ids": ["plugins/testing"] + }, + { + "type": "subcategory", + "label": "Publishing", + "ids": [ + "plugins/publishing", + "plugins/publish-private", + "plugins/add-to-marketplace" + ] + } + ], + "Configuration": [ + "conf/index", + "conf/reading", + "conf/writing", + "conf/defining" + ], + "Auth and identity": [ + "auth/index", + "auth/add-auth-provider", + "auth/auth-backend", + "auth/oauth", + "auth/glossary", + "auth/auth-backend-classes" + ], + + "Designing for Backstage": [ + "dls/design", + "dls/contributing-to-storybook", + "dls/figma" + ], + "API references": [ + { + "type": "subcategory", + "label": "TypeScript API", + "ids": [ + "api/utility-apis", + "reference/utility-apis/README", + "reference/createPlugin", + "reference/createPlugin-feature-flags", + "reference/createPlugin-router" + ] + }, + { + "type": "subcategory", + "label": "Backend APIs", + "ids": ["api/backend"] + } + ], + "Tutorials": [ + "tutorials/journey", + "tutorials/quickstart-app-auth", + "tutorials/quickstart-app-plugin" + ], + "Architecture Decision Records (ADRs)": [ + "architecture-decisions/adrs-overview", + "architecture-decisions/adrs-adr001", + "architecture-decisions/adrs-adr002", + "architecture-decisions/adrs-adr003", + "architecture-decisions/adrs-adr004", + "architecture-decisions/adrs-adr005", + "architecture-decisions/adrs-adr006", + "architecture-decisions/adrs-adr007", + "architecture-decisions/adrs-adr008" + ], + "Contribute": ["../CONTRIBUTING"], + "Support": ["overview/support"], + "FAQ": ["FAQ"] + } +} diff --git a/microsite/siteConfig.js b/microsite/siteConfig.js new file mode 100644 index 0000000000..d7a3bd67d0 --- /dev/null +++ b/microsite/siteConfig.js @@ -0,0 +1,128 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +// See https://docusaurus.io/docs/site-config for all the possible +// site configuration options. + +// List of projects/orgs using your project for the users page. +const users = []; + +const siteConfig = { + title: 'Backstage', // Title for your website. + tagline: 'An open platform for building developer portals', + url: 'https://backstage.io', // Your website URL + cname: 'backstage.io', + baseUrl: '/', // Base URL for your project */ + editUrl: 'https://github.com/spotify/backstage/edit/master/docs/', + + // Used for publishing and more + projectName: 'backstage', + organizationName: 'Spotify', + fossWebsite: 'https://spotify.github.io/', + + // Google Analytics + gaTrackingId: 'UA-48912878-10', + + // For no header links in the top nav bar -> headerLinks: [], + headerLinks: [ + { + href: 'https://github.com/spotify/backstage', + label: 'GitHub', + }, + { + doc: 'overview/what-is-backstage', + href: '/docs', + label: 'Docs', + }, + { + page: 'plugins', + label: 'Plugins', + }, + { + page: 'blog', + blog: true, + label: 'Blog', + }, + { + page: 'demos', + label: 'Demos', + }, + { + href: 'https://mailchi.mp/spotify/backstage-community', + label: 'Newsletter', + }, + ], + + /* path to images for header/footer */ + // headerIcon: "img/android-chrome-192x192.png", + footerIcon: 'img/android-chrome-192x192.png', + favicon: 'img/favicon.svg', + + /* Colors for website */ + colors: { + primaryColor: '#36BAA2', + secondaryColor: '#121212', + textColor: '#FFFFFF', + navigatorTitleTextColor: '#e4e4e4', + navigatorItemTextColor: '#9e9e9e', + navGroupSubcategoryTitleColor: '#9e9e9e', + }, + + /* Colors for syntax highlighting */ + highlight: { + theme: 'dark', + }, + + // This copyright info is used in /core/Footer.js and blog RSS/Atom feeds. + copyright: `Copyright © ${new Date().getFullYear()} Spotify AB`, + + highlight: { + // Highlight.js theme to use for syntax highlighting in code blocks. + theme: 'monokai', + }, + + // Add custom scripts here that would be placed in + + + `); +}; + +export const ensuresXRequestedWith = (req: express.Request) => { + const requiredHeader = req.header('X-Requested-With'); + + if (!requiredHeader || requiredHeader !== 'XMLHttpRequest') { + return false; + } + return true; +}; diff --git a/plugins/auth-backend/src/lib/flow/index.ts b/plugins/auth-backend/src/lib/flow/index.ts new file mode 100644 index 0000000000..a5f2f7a3ac --- /dev/null +++ b/plugins/auth-backend/src/lib/flow/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ensuresXRequestedWith, postMessageResponse } from './authFlowHelpers'; diff --git a/plugins/circleci/src/state/types.ts b/plugins/auth-backend/src/lib/flow/types.ts similarity index 59% rename from plugins/circleci/src/state/types.ts rename to plugins/auth-backend/src/lib/flow/types.ts index 41b3577082..98bb551c2c 100644 --- a/plugins/circleci/src/state/types.ts +++ b/plugins/auth-backend/src/lib/flow/types.ts @@ -14,23 +14,18 @@ * limitations under the License. */ -export type Settings = { owner: string; repo: string; token: string }; -export type SettingsState = Settings & { - showSettings: boolean; -}; +import { AuthResponse } from '../../providers/types'; -export type State = SettingsState; - -type SettingsAction = +/** + * Payload sent as a post message after the auth request is complete. + * If successful then has a valid payload with Auth information else contains an error. + */ +export type WebMessageResponse = | { - type: 'setCredentials'; - payload: { - repo: string; - owner: string; - token: string; - }; + type: 'authorization_response'; + response: AuthResponse; } - | { type: 'showSettings' } - | { type: 'hideSettings' }; - -export type Action = SettingsAction; + | { + type: 'authorization_response'; + error: Error; + }; diff --git a/plugins/auth-backend/src/lib/OAuthProvider.test.ts b/plugins/auth-backend/src/lib/oauth/OAuthAdapter.test.ts similarity index 56% rename from plugins/auth-backend/src/lib/OAuthProvider.test.ts rename to plugins/auth-backend/src/lib/oauth/OAuthAdapter.test.ts index 3d631bd4ac..d2b31213f2 100644 --- a/plugins/auth-backend/src/lib/OAuthProvider.test.ts +++ b/plugins/auth-backend/src/lib/oauth/OAuthAdapter.test.ts @@ -15,15 +15,9 @@ */ import express from 'express'; -import { - ensuresXRequestedWith, - postMessageResponse, - THOUSAND_DAYS_MS, - TEN_MINUTES_MS, - verifyNonce, - OAuthProvider, -} from './OAuthProvider'; -import { WebMessageResponse, OAuthProviderHandlers } from '../providers/types'; +import { THOUSAND_DAYS_MS, TEN_MINUTES_MS, OAuthAdapter } from './OAuthAdapter'; +import { encodeState } from './helpers'; +import { OAuthHandlers } from './types'; const mockResponseData = { providerInfo: { @@ -40,146 +34,8 @@ const mockResponseData = { }, }; -describe('OAuthProvider Utils', () => { - describe('verifyNonce', () => { - it('should throw error if cookie nonce missing', () => { - const mockRequest = ({ - cookies: {}, - query: { - state: 'NONCE', - }, - } as unknown) as express.Request; - expect(() => { - verifyNonce(mockRequest, 'providera'); - }).toThrowError('Auth response is missing cookie nonce'); - }); - - it('should throw error if state nonce missing', () => { - const mockRequest = ({ - cookies: { - 'providera-nonce': 'NONCE', - }, - query: {}, - } as unknown) as express.Request; - expect(() => { - verifyNonce(mockRequest, 'providera'); - }).toThrowError('Auth response is missing state nonce'); - }); - - it('should throw error if nonce mismatch', () => { - const mockRequest = ({ - cookies: { - 'providera-nonce': 'NONCEA', - }, - query: { - state: 'NONCEB', - }, - } as unknown) as express.Request; - expect(() => { - verifyNonce(mockRequest, 'providera'); - }).toThrowError('Invalid nonce'); - }); - - it('should not throw any error if nonce matches', () => { - const mockRequest = ({ - cookies: { - 'providera-nonce': 'NONCE', - }, - query: { - state: 'NONCE', - }, - } as unknown) as express.Request; - expect(() => { - verifyNonce(mockRequest, 'providera'); - }).not.toThrow(); - }); - }); - - describe('postMessageResponse', () => { - const appOrigin = 'http://localhost:3000'; - it('should post a message back with payload success', () => { - const mockResponse = ({ - end: jest.fn().mockReturnThis(), - setHeader: jest.fn().mockReturnThis(), - } as unknown) as express.Response; - - const data: WebMessageResponse = { - type: 'authorization_response', - response: { - providerInfo: { - accessToken: 'ACCESS_TOKEN', - idToken: 'ID_TOKEN', - expiresInSeconds: 10, - scope: 'email', - }, - profile: { - email: 'foo@bar.com', - }, - backstageIdentity: { - id: 'a', - idToken: 'a.b.c', - }, - }, - }; - const jsonData = JSON.stringify(data); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); - - postMessageResponse(mockResponse, appOrigin, data); - expect(mockResponse.setHeader).toBeCalledTimes(2); - expect(mockResponse.end).toBeCalledTimes(1); - expect(mockResponse.end).toBeCalledWith( - expect.stringContaining(base64Data), - ); - }); - - it('should post a message back with payload error', () => { - const mockResponse = ({ - end: jest.fn().mockReturnThis(), - setHeader: jest.fn().mockReturnThis(), - } as unknown) as express.Response; - - const data: WebMessageResponse = { - type: 'authorization_response', - error: new Error('Unknown error occured'), - }; - const jsonData = JSON.stringify(data); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); - - postMessageResponse(mockResponse, appOrigin, data); - expect(mockResponse.setHeader).toBeCalledTimes(2); - expect(mockResponse.end).toBeCalledTimes(1); - expect(mockResponse.end).toBeCalledWith( - expect.stringContaining(base64Data), - ); - }); - }); - - describe('ensuresXRequestedWith', () => { - it('should return false if no header present', () => { - const mockRequest = ({ - header: () => jest.fn(), - } as unknown) as express.Request; - expect(ensuresXRequestedWith(mockRequest)).toBe(false); - }); - - it('should return false if header present with incorrect value', () => { - const mockRequest = ({ - header: () => 'INVALID', - } as unknown) as express.Request; - expect(ensuresXRequestedWith(mockRequest)).toBe(false); - }); - - it('should return true if header present with correct value', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - } as unknown) as express.Request; - expect(ensuresXRequestedWith(mockRequest)).toBe(true); - }); - }); -}); - -describe('OAuthProvider', () => { - class MyAuthProvider implements OAuthProviderHandlers { +describe('OAuthAdapter', () => { + class MyAuthProvider implements OAuthHandlers { async start() { return { url: '/url', @@ -201,8 +57,9 @@ describe('OAuthProvider', () => { providerId: 'test-provider', secure: false, disableRefresh: true, - baseUrl: 'http://localhost:7000/auth', appOrigin: 'http://localhost:3000', + cookieDomain: 'localhost', + cookiePath: '/auth/test-provider', tokenIssuer: { issueToken: async () => 'my-id-token', listPublicKeys: async () => ({ keys: [] }), @@ -210,13 +67,14 @@ describe('OAuthProvider', () => { }; it('sets the correct headers in start', async () => { - const oauthProvider = new OAuthProvider( + const oauthProvider = new OAuthAdapter( providerInstance, oAuthProviderOptions, ); const mockRequest = ({ query: { scope: 'user', + env: 'development', }, } as unknown) as express.Request; @@ -244,17 +102,18 @@ describe('OAuthProvider', () => { }); it('sets the refresh cookie if refresh is enabled', async () => { - const oauthProvider = new OAuthProvider(providerInstance, { + const oauthProvider = new OAuthAdapter(providerInstance, { ...oAuthProviderOptions, disableRefresh: false, }); + const state = { nonce: 'nonce', env: 'development' }; const mockRequest = ({ cookies: { 'test-provider-nonce': 'nonce', }, query: { - state: 'nonce', + state: encodeState(state), }, } as unknown) as express.Request; @@ -277,7 +136,7 @@ describe('OAuthProvider', () => { }); it('does not set the refresh cookie if refresh is disabled', async () => { - const oauthProvider = new OAuthProvider(providerInstance, { + const oauthProvider = new OAuthAdapter(providerInstance, { ...oAuthProviderOptions, disableRefresh: true, }); @@ -302,7 +161,7 @@ describe('OAuthProvider', () => { }); it('removes refresh cookie when logging out', async () => { - const oauthProvider = new OAuthProvider(providerInstance, { + const oauthProvider = new OAuthAdapter(providerInstance, { ...oAuthProviderOptions, disableRefresh: false, }); @@ -327,7 +186,7 @@ describe('OAuthProvider', () => { it('gets new access-token when refreshing', async () => { oAuthProviderOptions.disableRefresh = false; - const oauthProvider = new OAuthProvider(providerInstance, { + const oauthProvider = new OAuthAdapter(providerInstance, { ...oAuthProviderOptions, disableRefresh: false, }); @@ -356,7 +215,7 @@ describe('OAuthProvider', () => { }); it('handles refresh without capabilities', async () => { - const oauthProvider = new OAuthProvider(providerInstance, { + const oauthProvider = new OAuthAdapter(providerInstance, { ...oAuthProviderOptions, disableRefresh: true, }); diff --git a/plugins/auth-backend/src/lib/OAuthProvider.ts b/plugins/auth-backend/src/lib/oauth/OAuthAdapter.ts similarity index 67% rename from plugins/auth-backend/src/lib/OAuthProvider.ts rename to plugins/auth-backend/src/lib/oauth/OAuthAdapter.ts index c4996c31a2..62cb8e444d 100644 --- a/plugins/auth-backend/src/lib/OAuthProvider.ts +++ b/plugins/auth-backend/src/lib/oauth/OAuthAdapter.ts @@ -19,12 +19,14 @@ import crypto from 'crypto'; import { URL } from 'url'; import { AuthProviderRouteHandlers, - OAuthProviderHandlers, - WebMessageResponse, BackstageIdentity, -} from '../providers/types'; + AuthProviderConfig, +} from '../../providers/types'; import { InputError } from '@backstage/backend-common'; -import { TokenIssuer } from '../identity'; +import { TokenIssuer } from '../../identity'; +import { verifyNonce } from './helpers'; +import { postMessageResponse, ensuresXRequestedWith } from '../flow'; +import { OAuthHandlers, OAuthStartRequest, OAuthRefreshRequest } from './types'; export const THOUSAND_DAYS_MS = 1000 * 24 * 60 * 60 * 1000; export const TEN_MINUTES_MS = 600 * 1000; @@ -34,78 +36,46 @@ export type Options = { secure: boolean; disableRefresh?: boolean; persistScopes?: boolean; - baseUrl: string; + cookieDomain: string; + cookiePath: string; appOrigin: string; tokenIssuer: TokenIssuer; }; -export const verifyNonce = (req: express.Request, providerId: string) => { - const cookieNonce = req.cookies[`${providerId}-nonce`]; - const stateNonce = req.query.state; - - if (!cookieNonce) { - throw new Error('Auth response is missing cookie nonce'); +export class OAuthAdapter implements AuthProviderRouteHandlers { + static fromConfig( + config: AuthProviderConfig, + handlers: OAuthHandlers, + options: Pick< + Options, + 'providerId' | 'persistScopes' | 'disableRefresh' | 'tokenIssuer' + >, + ): OAuthAdapter { + const { origin: appOrigin } = new URL(config.appUrl); + const secure = config.baseUrl.startsWith('https://'); + const url = new URL(config.baseUrl); + const cookiePath = `${url.pathname}/${options.providerId}`; + return new OAuthAdapter(handlers, { + ...options, + appOrigin, + cookieDomain: url.hostname, + cookiePath, + secure, + }); } - if (!stateNonce) { - throw new Error('Auth response is missing state nonce'); - } - if (cookieNonce !== stateNonce) { - throw new Error('Invalid nonce'); - } -}; - -export const postMessageResponse = ( - res: express.Response, - appOrigin: string, - response: WebMessageResponse, -) => { - const jsonData = JSON.stringify(response); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); - - res.setHeader('Content-Type', 'text/html'); - res.setHeader('X-Frame-Options', 'sameorigin'); - - // TODO: Make target app origin configurable globally - res.end(` - - - - - - `); -}; - -export const ensuresXRequestedWith = (req: express.Request) => { - const requiredHeader = req.header('X-Requested-With'); - - if (!requiredHeader || requiredHeader !== 'XMLHttpRequest') { - return false; - } - return true; -}; - -export class OAuthProvider implements AuthProviderRouteHandlers { - private readonly domain: string; - private readonly basePath: string; constructor( - private readonly providerHandlers: OAuthProviderHandlers, + private readonly handlers: OAuthHandlers, private readonly options: Options, - ) { - const url = new URL(options.baseUrl); - this.domain = url.hostname; - this.basePath = url.pathname; - } + ) {} async start(req: express.Request, res: express.Response): Promise { // retrieve scopes from request const scope = req.query.scope?.toString() ?? ''; + const env = req.query.env?.toString(); - if (!scope) { - throw new InputError('missing scope parameter'); + if (!env) { + throw new InputError('No env provided in request query parameters'); } if (this.options.persistScopes) { @@ -116,14 +86,11 @@ export class OAuthProvider implements AuthProviderRouteHandlers { // set a nonce cookie before redirecting to oauth provider this.setNonceCookie(res, nonce); - const queryParameters = { - scope, - state: nonce, - }; + const state = { nonce: nonce, env: env }; + const forwardReq = Object.assign(req, { scope, state }); - const { url, status } = await this.providerHandlers.start( - req, - queryParameters, + const { url, status } = await this.handlers.start( + forwardReq as OAuthStartRequest, ); res.statusCode = status || 302; @@ -140,9 +107,7 @@ export class OAuthProvider implements AuthProviderRouteHandlers { // verify nonce cookie and state cookie on callback verifyNonce(req, this.options.providerId); - const { response, refreshToken } = await this.providerHandlers.handler( - req, - ); + const { response, refreshToken } = await this.handlers.handler(req); if (this.options.persistScopes) { const grantedScopes = this.getScopesFromCookie( @@ -199,7 +164,7 @@ export class OAuthProvider implements AuthProviderRouteHandlers { return; } - if (!this.providerHandlers.refresh || this.options.disableRefresh) { + if (!this.handlers.refresh || this.options.disableRefresh) { res.send( `Refresh token not supported for provider: ${this.options.providerId}`, ); @@ -217,11 +182,22 @@ export class OAuthProvider implements AuthProviderRouteHandlers { const scope = req.query.scope?.toString() ?? ''; + const forwardReq = Object.assign(req, { scope, refreshToken }); + // get new access_token - const response = await this.providerHandlers.refresh(refreshToken, scope); + const response = await this.handlers.refresh( + forwardReq as OAuthRefreshRequest, + ); await this.populateIdentity(response.backstageIdentity); + if ( + response.providerInfo.refreshToken && + response.providerInfo.refreshToken !== refreshToken + ) { + this.setRefreshTokenCookie(res, response.providerInfo.refreshToken); + } + res.send(response); } catch (error) { res.status(401).send(`${error.message}`); @@ -248,9 +224,9 @@ export class OAuthProvider implements AuthProviderRouteHandlers { res.cookie(`${this.options.providerId}-nonce`, nonce, { maxAge: TEN_MINUTES_MS, secure: this.options.secure, - sameSite: 'none', - domain: this.domain, - path: `${this.basePath}/${this.options.providerId}/handler`, + sameSite: 'lax', + domain: this.options.cookieDomain, + path: `${this.options.cookiePath}/handler`, httpOnly: true, }); }; @@ -259,9 +235,9 @@ export class OAuthProvider implements AuthProviderRouteHandlers { res.cookie(`${this.options.providerId}-scope`, scope, { maxAge: TEN_MINUTES_MS, secure: this.options.secure, - sameSite: 'none', - domain: this.domain, - path: `${this.basePath}/${this.options.providerId}/handler`, + sameSite: 'lax', + domain: this.options.cookieDomain, + path: `${this.options.cookiePath}/handler`, httpOnly: true, }); }; @@ -277,9 +253,9 @@ export class OAuthProvider implements AuthProviderRouteHandlers { res.cookie(`${this.options.providerId}-refresh-token`, refreshToken, { maxAge: THOUSAND_DAYS_MS, secure: this.options.secure, - sameSite: 'none', - domain: this.domain, - path: `${this.basePath}/${this.options.providerId}`, + sameSite: 'lax', + domain: this.options.cookieDomain, + path: this.options.cookiePath, httpOnly: true, }); }; @@ -287,10 +263,10 @@ export class OAuthProvider implements AuthProviderRouteHandlers { private removeRefreshTokenCookie = (res: express.Response) => { res.cookie(`${this.options.providerId}-refresh-token`, '', { maxAge: 0, - secure: false, - sameSite: 'none', - domain: `${this.domain}`, - path: `${this.basePath}/${this.options.providerId}`, + secure: this.options.secure, + sameSite: 'lax', + domain: this.options.cookieDomain, + path: this.options.cookiePath, httpOnly: true, }); }; diff --git a/plugins/auth-backend/src/lib/oauth/OAuthEnvironmentHandler.ts b/plugins/auth-backend/src/lib/oauth/OAuthEnvironmentHandler.ts new file mode 100644 index 0000000000..d22fc52499 --- /dev/null +++ b/plugins/auth-backend/src/lib/oauth/OAuthEnvironmentHandler.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import { Config } from '@backstage/config'; +import { InputError } from '@backstage/backend-common'; +import { readState } from './helpers'; +import { AuthProviderRouteHandlers } from '../../providers/types'; + +export class OAuthEnvironmentHandler implements AuthProviderRouteHandlers { + static mapConfig( + config: Config, + factoryFunc: (envConfig: Config) => AuthProviderRouteHandlers, + ) { + const envs = config.keys(); + const handlers = new Map(); + + for (const env of envs) { + const envConfig = config.getConfig(env); + const handler = factoryFunc(envConfig); + handlers.set(env, handler); + } + + return new OAuthEnvironmentHandler(handlers); + } + + constructor( + private readonly handlers: Map, + ) {} + + async start(req: express.Request, res: express.Response): Promise { + const provider = this.getProviderForEnv(req, res); + await provider?.start(req, res); + } + + async frameHandler( + req: express.Request, + res: express.Response, + ): Promise { + const provider = this.getProviderForEnv(req, res); + await provider?.frameHandler(req, res); + } + + async refresh(req: express.Request, res: express.Response): Promise { + const provider = this.getProviderForEnv(req, res); + await provider?.refresh?.(req, res); + } + + async logout(req: express.Request, res: express.Response): Promise { + const provider = this.getProviderForEnv(req, res); + await provider?.logout?.(req, res); + } + + private getRequestFromEnv(req: express.Request): string | undefined { + const reqEnv = req.query.env?.toString(); + if (reqEnv) { + return reqEnv; + } + const stateParams = req.query.state?.toString(); + if (!stateParams) { + return undefined; + } + const env = readState(stateParams).env; + return env; + } + + private getProviderForEnv( + req: express.Request, + res: express.Response, + ): AuthProviderRouteHandlers | undefined { + const env: string | undefined = this.getRequestFromEnv(req); + + if (!env) { + throw new InputError(`Must specify 'env' query to select environment`); + } + + if (!this.handlers.has(env)) { + res.status(404).send( + `Missing configuration. +
+
+ For this flow to work you need to supply a valid configuration for the "${env}" environment of provider.`, + ); + return undefined; + } + + return this.handlers.get(env); + } +} diff --git a/plugins/auth-backend/src/lib/oauth/helpers.test.ts b/plugins/auth-backend/src/lib/oauth/helpers.test.ts new file mode 100644 index 0000000000..fcf56705d2 --- /dev/null +++ b/plugins/auth-backend/src/lib/oauth/helpers.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import { verifyNonce, encodeState } from './helpers'; + +describe('OAuthProvider Utils', () => { + describe('verifyNonce', () => { + it('should throw error if cookie nonce missing', () => { + const state = { nonce: 'NONCE', env: 'development' }; + const mockRequest = ({ + cookies: {}, + query: { + state: encodeState(state), + }, + } as unknown) as express.Request; + expect(() => { + verifyNonce(mockRequest, 'providera'); + }).toThrowError('Auth response is missing cookie nonce'); + }); + + it('should throw error if state nonce missing', () => { + const mockRequest = ({ + cookies: { + 'providera-nonce': 'NONCE', + }, + query: {}, + } as unknown) as express.Request; + expect(() => { + verifyNonce(mockRequest, 'providera'); + }).toThrowError('Invalid state passed via request'); + }); + + it('should throw error if nonce mismatch', () => { + const state = { nonce: 'NONCEB', env: 'development' }; + const mockRequest = ({ + cookies: { + 'providera-nonce': 'NONCEA', + }, + query: { + state: encodeState(state), + }, + } as unknown) as express.Request; + expect(() => { + verifyNonce(mockRequest, 'providera'); + }).toThrowError('Invalid nonce'); + }); + + it('should not throw any error if nonce matches', () => { + const state = { nonce: 'NONCE', env: 'development' }; + const mockRequest = ({ + cookies: { + 'providera-nonce': 'NONCE', + }, + query: { + state: encodeState(state), + }, + } as unknown) as express.Request; + expect(() => { + verifyNonce(mockRequest, 'providera'); + }).not.toThrow(); + }); + }); +}); diff --git a/plugins/auth-backend/src/lib/oauth/helpers.ts b/plugins/auth-backend/src/lib/oauth/helpers.ts new file mode 100644 index 0000000000..9f250a0285 --- /dev/null +++ b/plugins/auth-backend/src/lib/oauth/helpers.ts @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import { OAuthState } from './types'; + +export const readState = (stateString: string): OAuthState => { + const state = Object.fromEntries( + new URLSearchParams(decodeURIComponent(stateString)), + ); + if ( + !state.nonce || + !state.env || + state.nonce?.length === 0 || + state.env?.length === 0 + ) { + throw Error(`Invalid state passed via request`); + } + return { + nonce: state.nonce, + env: state.env, + }; +}; + +export const encodeState = (state: OAuthState): string => { + const searchParams = new URLSearchParams(); + searchParams.append('nonce', state.nonce); + searchParams.append('env', state.env); + + return encodeURIComponent(searchParams.toString()); +}; + +export const verifyNonce = (req: express.Request, providerId: string) => { + const cookieNonce = req.cookies[`${providerId}-nonce`]; + const state: OAuthState = readState(req.query.state?.toString() ?? ''); + const stateNonce = state.nonce; + + if (!cookieNonce) { + throw new Error('Auth response is missing cookie nonce'); + } + if (stateNonce.length === 0) { + throw new Error('Auth response is missing state nonce'); + } + if (cookieNonce !== stateNonce) { + throw new Error('Invalid nonce'); + } +}; diff --git a/plugins/auth-backend/src/lib/oauth/index.ts b/plugins/auth-backend/src/lib/oauth/index.ts new file mode 100644 index 0000000000..564a0e1e7c --- /dev/null +++ b/plugins/auth-backend/src/lib/oauth/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { OAuthEnvironmentHandler } from './OAuthEnvironmentHandler'; +export { OAuthAdapter } from './OAuthAdapter'; +export { encodeState } from './helpers'; +export type { + OAuthHandlers, + OAuthProviderInfo, + OAuthProviderOptions, + OAuthResponse, + OAuthState, + OAuthStartRequest, + OAuthRefreshRequest, +} from './types'; diff --git a/plugins/auth-backend/src/lib/oauth/types.ts b/plugins/auth-backend/src/lib/oauth/types.ts new file mode 100644 index 0000000000..b2b7915a4e --- /dev/null +++ b/plugins/auth-backend/src/lib/oauth/types.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import { AuthResponse, RedirectInfo } from '../../providers/types'; + +/** + * Common options for passport.js-based OAuth providers + */ +export type OAuthProviderOptions = { + /** + * Client ID of the auth provider. + */ + clientId: string; + /** + * Client Secret of the auth provider. + */ + clientSecret: string; + /** + * Callback URL to be passed to the auth provider to redirect to after the user signs in. + */ + callbackUrl: string; +}; + +export type OAuthResponse = AuthResponse; + +export type OAuthProviderInfo = { + /** + * An access token issued for the signed in user. + */ + accessToken: string; + /** + * (Optional) Id token issued for the signed in user. + */ + idToken?: string; + /** + * Expiry of the access token in seconds. + */ + expiresInSeconds?: number; + /** + * Scopes granted for the access token. + */ + scope: string; + /** + * A refresh token issued for the signed in user + */ + refreshToken?: string; +}; + +export type OAuthState = { + /* A type for the serialized value in the `state` parameter of the OAuth authorization flow + */ + nonce: string; + env: string; +}; + +export type OAuthStartRequest = express.Request<{}> & { + scope: string; + state: OAuthState; +}; + +export type OAuthRefreshRequest = express.Request<{}> & { + scope: string; + refreshToken: string; +}; + +/** + * Any OAuth provider needs to implement this interface which has provider specific + * handlers for different methods to perform authentication, get access tokens, + * refresh tokens and perform sign out. + */ +export interface OAuthHandlers { + /** + * This method initiates a sign in request with an auth provider. + * @param {express.Request} req + * @param options + */ + start(req: OAuthStartRequest): Promise; + + /** + * Handles the redirect from the auth provider when the user has signed in. + * @param {express.Request} req + */ + handler( + req: express.Request, + ): Promise<{ + response: AuthResponse; + refreshToken?: string; + }>; + + /** + * (Optional) Given a refresh token and scope fetches a new access token from the auth provider. + * @param {string} refreshToken + * @param {string} scope + */ + refresh?(req: OAuthRefreshRequest): Promise>; + + /** + * (Optional) Sign out of the auth provider. + */ + logout?(): Promise; +} diff --git a/plugins/auth-backend/src/lib/PassportStrategyHelper.test.ts b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts similarity index 100% rename from plugins/auth-backend/src/lib/PassportStrategyHelper.test.ts rename to plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts diff --git a/plugins/auth-backend/src/lib/PassportStrategyHelper.ts b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts similarity index 88% rename from plugins/auth-backend/src/lib/PassportStrategyHelper.ts rename to plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts index 4cc0de4444..7bc34186f4 100644 --- a/plugins/auth-backend/src/lib/PassportStrategyHelper.ts +++ b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts @@ -17,12 +17,13 @@ import express from 'express'; import passport from 'passport'; import jwtDecoder from 'jwt-decode'; -import { - RedirectInfo, - RefreshTokenResponse, - ProfileInfo, - ProviderStrategy, -} from '../providers/types'; +import { ProfileInfo, RedirectInfo } from '../../providers/types'; + +export type PassportDoneCallback = ( + err?: Error, + response?: Res, + privateInfo?: Private, +) => void; export const makeProfileInfo = ( profile: passport.Profile, @@ -45,7 +46,6 @@ export const makeProfileInfo = ( if ((!email || !picture) && idToken) { try { const decoded: Record = jwtDecoder(idToken); - if (!email && decoded.email) { email = decoded.email; } @@ -107,6 +107,18 @@ export const executeFrameHandlerStrategy = async ( ); }; +type RefreshTokenResponse = { + /** + * An access token issued for the signed in user. + */ + accessToken: string; + /** + * Optionally, the server can issue a new Refresh Token for the user + */ + refreshToken?: string; + params: any; +}; + export const executeRefreshTokenStrategy = async ( providerStrategy: passport.Strategy, refreshToken: string, @@ -133,7 +145,7 @@ export const executeRefreshTokenStrategy = async ( ( err: Error | null, accessToken: string, - _refreshToken: string, + newRefreshToken: string, params: any, ) => { if (err) { @@ -149,6 +161,7 @@ export const executeRefreshTokenStrategy = async ( resolve({ accessToken, + refreshToken: newRefreshToken, params, }); }, @@ -156,6 +169,10 @@ export const executeRefreshTokenStrategy = async ( }); }; +type ProviderStrategy = { + userProfile(accessToken: string, callback: Function): void; +}; + export const executeFetchUserProfileStrategy = async ( providerStrategy: passport.Strategy, accessToken: string, diff --git a/plugins/auth-backend/src/lib/passport/index.ts b/plugins/auth-backend/src/lib/passport/index.ts new file mode 100644 index 0000000000..c307e212fa --- /dev/null +++ b/plugins/auth-backend/src/lib/passport/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + executeFetchUserProfileStrategy, + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, + makeProfileInfo, +} from './PassportStrategyHelper'; +export type { PassportDoneCallback } from './PassportStrategyHelper'; diff --git a/plugins/auth-backend/src/providers/auth0/index.ts b/plugins/auth-backend/src/providers/auth0/index.ts new file mode 100644 index 0000000000..87c9aceaa4 --- /dev/null +++ b/plugins/auth-backend/src/providers/auth0/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createAuth0Provider } from './provider'; diff --git a/plugins/auth-backend/src/providers/auth0/provider.ts b/plugins/auth-backend/src/providers/auth0/provider.ts new file mode 100644 index 0000000000..668ab17ee1 --- /dev/null +++ b/plugins/auth-backend/src/providers/auth0/provider.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import passport from 'passport'; +import Auth0Strategy from './strategy'; +import { + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; +import { + executeFetchUserProfileStrategy, + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, + makeProfileInfo, + PassportDoneCallback, +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; + +type PrivateInfo = { + refreshToken: string; +}; + +export type Auth0AuthProviderOptions = OAuthProviderOptions & { + domain: string; +}; + +export class Auth0AuthProvider implements OAuthHandlers { + private readonly _strategy: Auth0Strategy; + + constructor(options: Auth0AuthProviderOptions) { + this._strategy = new Auth0Strategy( + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + domain: options.domain, + passReqToCallback: false as true, + }, + ( + accessToken: any, + refreshToken: any, + params: any, + rawProfile: passport.Profile, + done: PassportDoneCallback, + ) => { + const profile = makeProfileInfo(rawProfile, params.id_token); + done( + undefined, + { + providerInfo: { + idToken: params.id_token, + accessToken, + scope: params.scope, + expiresInSeconds: params.expires_in, + }, + profile, + }, + { + refreshToken, + }, + ); + }, + ); + } + + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { + accessType: 'offline', + prompt: 'consent', + scope: req.scope, + state: encodeState(req.state), + }); + } + + async handler( + req: express.Request, + ): Promise<{ response: OAuthResponse; refreshToken: string }> { + const { response, privateInfo } = await executeFrameHandlerStrategy< + OAuthResponse, + PrivateInfo + >(req, this._strategy); + + return { + response: await this.populateIdentity(response), + refreshToken: privateInfo.refreshToken, + }; + } + + async refresh(req: OAuthRefreshRequest): Promise { + const { accessToken, params } = await executeRefreshTokenStrategy( + this._strategy, + req.refreshToken, + req.scope, + ); + + const profile = await executeFetchUserProfileStrategy( + this._strategy, + accessToken, + params.id_token, + ); + + return this.populateIdentity({ + providerInfo: { + accessToken, + idToken: params.id_token, + expiresInSeconds: params.expires_in, + scope: params.scope, + }, + profile, + }); + } + + // Use this function to grab the user profile info from the token + // Then populate the profile with it + private async populateIdentity( + response: OAuthResponse, + ): Promise { + const { profile } = response; + + if (!profile.email) { + throw new Error('Profile does not contain a profile'); + } + + const id = profile.email.split('@')[0]; + + return { ...response, backstageIdentity: { id } }; + } +} + +export const createAuth0Provider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'auth0'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const domain = envConfig.getString('domain'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + + const provider = new Auth0AuthProvider({ + clientId, + clientSecret, + callbackUrl, + domain, + }); + + return OAuthAdapter.fromConfig(globalConfig, provider, { + disableRefresh: true, + providerId, + tokenIssuer, + }); + }); diff --git a/plugins/auth-backend/src/providers/auth0/strategy.ts b/plugins/auth-backend/src/providers/auth0/strategy.ts new file mode 100644 index 0000000000..6ac06ec4e9 --- /dev/null +++ b/plugins/auth-backend/src/providers/auth0/strategy.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import OAuth2Strategy from 'passport-oauth2'; + +export interface Auth0StrategyOptionsWithRequest { + clientID: string; + clientSecret: string; + callbackURL: string; + domain: string; + passReqToCallback: true; +} + +export default class Auth0Strategy extends OAuth2Strategy { + constructor( + options: Auth0StrategyOptionsWithRequest, + verify: OAuth2Strategy.VerifyFunctionWithRequest, + ) { + const optionsWithURLs = { + ...options, + authorizationURL: `https://${options.domain}/authorize`, + tokenURL: `https://${options.domain}/oauth/token`, + userInfoURL: `https://${options.domain}/userinfo`, + apiUrl: `https://${options.domain}/api`, + }; + super(optionsWithURLs, verify); + } +} diff --git a/plugins/auth-backend/src/providers/factories.ts b/plugins/auth-backend/src/providers/factories.ts index 4c9caf214b..6c83dffda3 100644 --- a/plugins/auth-backend/src/providers/factories.ts +++ b/plugins/auth-backend/src/providers/factories.ts @@ -23,7 +23,10 @@ import { createGoogleProvider } from './google'; import { createOAuth2Provider } from './oauth2'; import { createOktaProvider } from './okta'; import { createSamlProvider } from './saml'; +import { createAuth0Provider } from './auth0'; +import { createMicrosoftProvider } from './microsoft'; import { AuthProviderConfig, AuthProviderFactory } from './types'; +import { Config } from '@backstage/config'; const factories: { [providerId: string]: AuthProviderFactory } = { google: createGoogleProvider, @@ -31,32 +34,35 @@ const factories: { [providerId: string]: AuthProviderFactory } = { gitlab: createGitlabProvider, saml: createSamlProvider, okta: createOktaProvider, + auth0: createAuth0Provider, + microsoft: createMicrosoftProvider, oauth2: createOAuth2Provider, }; export const createAuthProviderRouter = ( providerId: string, globalConfig: AuthProviderConfig, - providerConfig: any, // TODO: make this a config reader object of sorts + config: Config, logger: Logger, - issuer: TokenIssuer, + tokenIssuer: TokenIssuer, ) => { const factory = factories[providerId]; if (!factory) { throw Error(`No auth provider available for '${providerId}'`); } - const provider = factory(globalConfig, providerConfig, logger, issuer); - const router = Router(); - router.get('/start', provider.start.bind(provider)); - router.get('/handler/frame', provider.frameHandler.bind(provider)); - router.post('/handler/frame', provider.frameHandler.bind(provider)); - if (provider.logout) { - router.post('/logout', provider.logout.bind(provider)); + + const handler = factory({ globalConfig, config, logger, tokenIssuer }); + + router.get('/start', handler.start.bind(handler)); + router.get('/handler/frame', handler.frameHandler.bind(handler)); + router.post('/handler/frame', handler.frameHandler.bind(handler)); + if (handler.logout) { + router.post('/logout', handler.logout.bind(handler)); } - if (provider.refresh) { - router.get('/refresh', provider.refresh.bind(provider)); + if (handler.refresh) { + router.get('/refresh', handler.refresh.bind(handler)); } return router; diff --git a/plugins/auth-backend/src/providers/github/provider.ts b/plugins/auth-backend/src/providers/github/provider.ts index bd7d4aa712..bab7b3fc28 100644 --- a/plugins/auth-backend/src/providers/github/provider.ts +++ b/plugins/auth-backend/src/providers/github/provider.ts @@ -20,27 +20,27 @@ import { executeFrameHandlerStrategy, executeRedirectStrategy, makeProfileInfo, -} from '../../lib/PassportStrategyHelper'; -import { - OAuthProviderHandlers, - AuthProviderConfig, - RedirectInfo, - EnvironmentProviderConfig, - OAuthProviderOptions, - OAuthProviderConfig, - OAuthResponse, PassportDoneCallback, -} from '../types'; -import { OAuthProvider } from '../../lib/OAuthProvider'; +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; import { - EnvironmentHandlers, - EnvironmentHandler, -} from '../../lib/EnvironmentHandler'; -import { Logger } from 'winston'; -import { TokenIssuer } from '../../identity'; + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, +} from '../../lib/oauth'; import passport from 'passport'; -export class GithubAuthProvider implements OAuthProviderHandlers { +export type GithubAuthProviderOptions = OAuthProviderOptions & { + tokenUrl?: string; + userProfileUrl?: string; + authorizationUrl?: string; +}; + +export class GithubAuthProvider implements OAuthHandlers { private readonly _strategy: GithubStrategy; static transformPassportProfile(rawProfile: any): passport.Profile { @@ -73,7 +73,7 @@ export class GithubAuthProvider implements OAuthProviderHandlers { idToken: params.id_token, }; - // Github provides an id numeric value (123) + // GitHub provides an id numeric value (123) // as a fallback const id = passportProfile!.id; @@ -92,9 +92,16 @@ export class GithubAuthProvider implements OAuthProviderHandlers { }; } - constructor(options: OAuthProviderOptions) { + constructor(options: GithubAuthProviderOptions) { this._strategy = new GithubStrategy( - { ...options }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + tokenURL: options.tokenUrl, + userProfileURL: options.userProfileUrl, + authorizationURL: options.authorizationUrl, + }, ( accessToken: any, _: any, @@ -112,11 +119,11 @@ export class GithubAuthProvider implements OAuthProviderHandlers { ); } - async start( - req: express.Request, - options: Record, - ): Promise { - return await executeRedirectStrategy(req, this._strategy, options); + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { + scope: req.scope, + state: encodeState(req.state), + }); } async handler(req: express.Request) { @@ -129,46 +136,42 @@ export class GithubAuthProvider implements OAuthProviderHandlers { } } -export function createGithubProvider( - { baseUrl }: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const providerId = 'github'; - const envProviders: EnvironmentHandlers = {}; +export const createGithubProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'github'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const enterpriseInstanceUrl = envConfig.getOptionalString( + 'enterpriseInstanceUrl', + ); + const authorizationUrl = enterpriseInstanceUrl + ? `${enterpriseInstanceUrl}/login/oauth/authorize` + : undefined; + const tokenUrl = enterpriseInstanceUrl + ? `${enterpriseInstanceUrl}/login/oauth/access_token` + : undefined; + const userProfileUrl = enterpriseInstanceUrl + ? `${enterpriseInstanceUrl}/api/v3/user` + : undefined; + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - for (const [env, envConfig] of Object.entries(providerConfig)) { - const config = (envConfig as unknown) as OAuthProviderConfig; - const { secure, appOrigin } = config; - const opts = { - clientID: config.clientId, - clientSecret: config.clientSecret, - callbackURL: `${baseUrl}/${providerId}/handler/frame?env=${env}`, - }; + const provider = new GithubAuthProvider({ + clientId, + clientSecret, + callbackUrl, + tokenUrl, + userProfileUrl, + authorizationUrl, + }); - if (!opts.clientID || !opts.clientSecret) { - if (process.env.NODE_ENV !== 'development') { - throw new Error( - 'Failed to initialize Github auth provider, set AUTH_GITHUB_CLIENT_ID and AUTH_GITHUB_CLIENT_SECRET env vars', - ); - } - - logger.warn( - 'Github auth provider disabled, set AUTH_GITHUB_CLIENT_ID and AUTH_GITHUB_CLIENT_SECRET env vars to enable', - ); - continue; - } - - envProviders[env] = new OAuthProvider(new GithubAuthProvider(opts), { + return OAuthAdapter.fromConfig(globalConfig, provider, { disableRefresh: true, persistScopes: true, providerId, - secure, - baseUrl, - appOrigin, tokenIssuer, }); - } - return new EnvironmentHandler(providerId, envProviders); -} + }); diff --git a/plugins/auth-backend/src/providers/gitlab/provider.ts b/plugins/auth-backend/src/providers/gitlab/provider.ts index af1d57fcce..4d4ecc3b56 100644 --- a/plugins/auth-backend/src/providers/gitlab/provider.ts +++ b/plugins/auth-backend/src/providers/gitlab/provider.ts @@ -20,27 +20,25 @@ import { executeFrameHandlerStrategy, executeRedirectStrategy, makeProfileInfo, -} from '../../lib/PassportStrategyHelper'; -import { - OAuthProviderHandlers, - AuthProviderConfig, - RedirectInfo, - EnvironmentProviderConfig, - OAuthProviderOptions, - OAuthProviderConfig, - OAuthResponse, PassportDoneCallback, -} from '../types'; -import { OAuthProvider } from '../../lib/OAuthProvider'; +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; import { - EnvironmentHandlers, - EnvironmentHandler, -} from '../../lib/EnvironmentHandler'; -import { Logger } from 'winston'; -import { TokenIssuer } from '../../identity'; + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, +} from '../../lib/oauth'; import passport from 'passport'; -export class GitlabAuthProvider implements OAuthProviderHandlers { +export type GitlabAuthProviderOptions = OAuthProviderOptions & { + baseUrl: string; +}; + +export class GitlabAuthProvider implements OAuthHandlers { private readonly _strategy: GitlabStrategy; static transformPassportProfile(rawProfile: any): passport.Profile { @@ -101,9 +99,14 @@ export class GitlabAuthProvider implements OAuthProviderHandlers { }; } - constructor(options: OAuthProviderOptions) { + constructor(options: GitlabAuthProviderOptions) { this._strategy = new GitlabStrategy( - { ...options }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + baseURL: options.baseUrl, + }, ( accessToken: any, _: any, @@ -121,11 +124,11 @@ export class GitlabAuthProvider implements OAuthProviderHandlers { ); } - async start( - req: express.Request, - options: Record, - ): Promise { - return await executeRedirectStrategy(req, this._strategy, options); + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { + scope: req.scope, + state: encodeState(req.state), + }); } async handler(req: express.Request): Promise<{ response: OAuthResponse }> { @@ -136,51 +139,29 @@ export class GitlabAuthProvider implements OAuthProviderHandlers { } } -export function createGitlabProvider( - { baseUrl }: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const providerId = 'gitlab'; - const envProviders: EnvironmentHandlers = {}; +export const createGitlabProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'gitlab'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const audience = envConfig.getString('audience'); + const baseUrl = audience || 'https://gitlab.com'; + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - for (const [env, envConfig] of Object.entries(providerConfig)) { - const { - secure, - appOrigin, + const provider = new GitlabAuthProvider({ clientId, clientSecret, - audience, - } = (envConfig as unknown) as OAuthProviderConfig; - const opts = { - clientID: clientId, - clientSecret: clientSecret, - callbackURL: `${baseUrl}/${providerId}/handler/frame?env=${env}`, - baseURL: audience, - }; + callbackUrl, + baseUrl, + }); - if (!opts.clientID || !opts.clientSecret) { - if (process.env.NODE_ENV !== 'development') { - throw new Error( - 'Failed to initialize Gitlab auth provider, set AUTH_GITLAB_CLIENT_ID and AUTH_GITLAB_CLIENT_SECRET env vars', - ); - } - - logger.warn( - 'Gitlab auth provider disabled, set AUTH_GITLAB_CLIENT_ID and AUTH_GITLAB_CLIENT_SECRET env vars to enable', - ); - continue; - } - - envProviders[env] = new OAuthProvider(new GitlabAuthProvider(opts), { + return OAuthAdapter.fromConfig(globalConfig, provider, { disableRefresh: true, providerId, - secure, - baseUrl, - appOrigin, tokenIssuer, }); - } - return new EnvironmentHandler(providerId, envProviders); -} + }); diff --git a/plugins/auth-backend/src/providers/google/provider.ts b/plugins/auth-backend/src/providers/google/provider.ts index 3023c09886..3cc585605c 100644 --- a/plugins/auth-backend/src/providers/google/provider.ts +++ b/plugins/auth-backend/src/providers/google/provider.ts @@ -22,39 +22,39 @@ import { executeRefreshTokenStrategy, makeProfileInfo, executeFetchUserProfileStrategy, -} from '../../lib/PassportStrategyHelper'; -import { - OAuthProviderHandlers, - RedirectInfo, - AuthProviderConfig, - EnvironmentProviderConfig, - OAuthProviderOptions, - OAuthProviderConfig, - OAuthResponse, PassportDoneCallback, -} from '../types'; -import { OAuthProvider } from '../../lib/OAuthProvider'; -import passport from 'passport'; +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; import { - EnvironmentHandler, - EnvironmentHandlers, -} from '../../lib/EnvironmentHandler'; -import { Logger } from 'winston'; -import { TokenIssuer } from '../../identity'; + OAuthAdapter, + OAuthHandlers, + OAuthProviderOptions, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; +import passport from 'passport'; type PrivateInfo = { refreshToken: string; }; -export class GoogleAuthProvider implements OAuthProviderHandlers { +export class GoogleAuthProvider implements OAuthHandlers { private readonly _strategy: GoogleStrategy; constructor(options: OAuthProviderOptions) { // TODO: throw error if env variables not set? this._strategy = new GoogleStrategy( - // We need passReqToCallback set to false to get params, but there's - // no matching type signature for that, so instead behold this beauty - { ...options, passReqToCallback: false as true }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + // We need passReqToCallback set to false to get params, but there's + // no matching type signature for that, so instead behold this beauty + passReqToCallback: false as true, + }, ( accessToken: any, refreshToken: any, @@ -82,16 +82,13 @@ export class GoogleAuthProvider implements OAuthProviderHandlers { ); } - async start( - req: express.Request, - options: Record, - ): Promise { - const providerOptions = { - ...options, + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { accessType: 'offline', prompt: 'consent', - }; - return await executeRedirectStrategy(req, this._strategy, providerOptions); + scope: req.scope, + state: encodeState(req.state), + }); } async handler( @@ -108,11 +105,11 @@ export class GoogleAuthProvider implements OAuthProviderHandlers { }; } - async refresh(refreshToken: string, scope: string): Promise { + async refresh(req: OAuthRefreshRequest): Promise { const { accessToken, params } = await executeRefreshTokenStrategy( this._strategy, - refreshToken, - scope, + req.refreshToken, + req.scope, ); const profile = await executeFetchUserProfileStrategy( @@ -148,45 +145,26 @@ export class GoogleAuthProvider implements OAuthProviderHandlers { } } -export function createGoogleProvider( - { baseUrl }: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const providerId = 'google'; - const envProviders: EnvironmentHandlers = {}; +export const createGoogleProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'google'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - for (const [env, envConfig] of Object.entries(providerConfig)) { - const config = (envConfig as unknown) as OAuthProviderConfig; - const { secure, appOrigin } = config; - const opts = { - clientID: config.clientId, - clientSecret: config.clientSecret, - callbackURL: `${baseUrl}/${providerId}/handler/frame?env=${env}`, - }; + const provider = new GoogleAuthProvider({ + clientId, + clientSecret, + callbackUrl, + }); - if (!opts.clientID || !opts.clientSecret) { - if (process.env.NODE_ENV !== 'development') { - throw new Error( - 'Failed to initialize Google auth provider, set AUTH_GOOGLE_CLIENT_ID and AUTH_GOOGLE_CLIENT_SECRET env vars', - ); - } - - logger.warn( - 'Google auth provider disabled, set AUTH_GOOGLE_CLIENT_ID and AUTH_GOOGLE_CLIENT_SECRET env vars to enable', - ); - continue; - } - - envProviders[env] = new OAuthProvider(new GoogleAuthProvider(opts), { + return OAuthAdapter.fromConfig(globalConfig, provider, { disableRefresh: false, providerId, - secure, - baseUrl, - appOrigin, tokenIssuer, }); - } - return new EnvironmentHandler(providerId, envProviders); -} + }); diff --git a/plugins/auth-backend/src/providers/microsoft/index.ts b/plugins/auth-backend/src/providers/microsoft/index.ts new file mode 100644 index 0000000000..2e4abd2d2c --- /dev/null +++ b/plugins/auth-backend/src/providers/microsoft/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createMicrosoftProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/microsoft/provider.ts b/plugins/auth-backend/src/providers/microsoft/provider.ts new file mode 100644 index 0000000000..baa66f0662 --- /dev/null +++ b/plugins/auth-backend/src/providers/microsoft/provider.ts @@ -0,0 +1,237 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import passport from 'passport'; +import { Strategy as MicrosoftStrategy } from 'passport-microsoft'; + +import { + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, + makeProfileInfo, + executeFetchUserProfileStrategy, + PassportDoneCallback, +} from '../../lib/passport'; + +import { RedirectInfo, AuthProviderFactory } from '../types'; + +import { + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; + +import got from 'got'; + +type PrivateInfo = { + refreshToken: string; +}; + +export type MicrosoftAuthProviderOptions = OAuthProviderOptions & { + authorizationUrl?: string; + tokenUrl?: string; +}; + +export class MicrosoftAuthProvider implements OAuthHandlers { + private readonly _strategy: MicrosoftStrategy; + + static transformAuthResponse( + accessToken: string, + params: any, + rawProfile: any, + photoURL: any, + ): OAuthResponse { + let passportProfile: passport.Profile = rawProfile; + passportProfile = { + ...passportProfile, + photos: [{ value: photoURL }], + }; + + const profile = makeProfileInfo(passportProfile, params.id_token); + const providerInfo = { + idToken: params.id_token, + accessToken, + scope: params.scope, + expiresInSeconds: params.expires_in, + }; + + return { + providerInfo, + profile, + }; + } + + constructor(options: MicrosoftAuthProviderOptions) { + this._strategy = new MicrosoftStrategy( + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + authorizationURL: options.authorizationUrl, + tokenURL: options.tokenUrl, + passReqToCallback: false as true, + }, + ( + accessToken: any, + refreshToken: any, + params: any, + rawProfile: passport.Profile, + done: PassportDoneCallback, + ) => { + this.getUserPhoto(accessToken) + .then(photoURL => { + const authResponse = MicrosoftAuthProvider.transformAuthResponse( + accessToken, + params, + rawProfile, + photoURL, + ); + done(undefined, authResponse, { refreshToken }); + }) + .catch(error => { + throw new Error(`Error processing auth response: ${error}`); + }); + }, + ); + } + + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { + scope: req.scope, + state: encodeState(req.state), + }); + } + + async handler( + req: express.Request, + ): Promise<{ response: OAuthResponse; refreshToken: string }> { + const { response, privateInfo } = await executeFrameHandlerStrategy< + OAuthResponse, + PrivateInfo + >(req, this._strategy); + + return { + response: await this.populateIdentity(response), + refreshToken: privateInfo.refreshToken, + }; + } + + async refresh(req: OAuthRefreshRequest): Promise { + const { accessToken, params } = await executeRefreshTokenStrategy( + this._strategy, + req.refreshToken, + req.scope, + ); + + const profile = await executeFetchUserProfileStrategy( + this._strategy, + accessToken, + params.id_token, + ); + const photo = await this.getUserPhoto(accessToken); + if (photo) { + profile.picture = photo; + } + + return this.populateIdentity({ + providerInfo: { + accessToken, + idToken: params.id_token, + expiresInSeconds: params.expires_in, + scope: params.scope, + }, + profile, + }); + } + + private getUserPhoto(accessToken: string): Promise { + return new Promise(resolve => { + got + .get('https://graph.microsoft.com/v1.0/me/photos/48x48/$value', { + encoding: 'binary', + responseType: 'buffer', + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + .then(photoData => { + const photoURL = `data:image/jpeg;base64,${Buffer.from( + photoData.body, + ).toString('base64')}`; + resolve(photoURL); + }) + .catch(error => { + console.log( + `Could not retrieve user profile photo from Microsoft Graph API: ${error}`, + ); + // User profile photo is optional, ignore errors and resolve undefined + resolve(); + }); + }); + } + + private async populateIdentity( + response: OAuthResponse, + ): Promise { + const { profile } = response; + + if (!profile.email) { + throw new Error('Microsoft profile contained no email'); + } + + // Like Google implementation, setting this to local part of email for now + const id = profile.email.split('@')[0]; + + return { ...response, backstageIdentity: { id } }; + } +} + +export const createMicrosoftProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'microsoft'; + + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const tenantID = envConfig.getString('tenantId'); + + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + const authorizationUrl = `https://login.microsoftonline.com/${tenantID}/oauth2/v2.0/authorize`; + const tokenUrl = `https://login.microsoftonline.com/${tenantID}/oauth2/v2.0/token`; + + const provider = new MicrosoftAuthProvider({ + clientId, + clientSecret, + callbackUrl, + authorizationUrl, + tokenUrl, + }); + + return OAuthAdapter.fromConfig(globalConfig, provider, { + disableRefresh: false, + providerId, + tokenIssuer, + }); + }); diff --git a/plugins/auth-backend/src/providers/oauth2/provider.ts b/plugins/auth-backend/src/providers/oauth2/provider.ts index 60b6bc605b..a657ea2195 100644 --- a/plugins/auth-backend/src/providers/oauth2/provider.ts +++ b/plugins/auth-backend/src/providers/oauth2/provider.ts @@ -17,41 +17,48 @@ import express from 'express'; import passport from 'passport'; import { Strategy as OAuth2Strategy } from 'passport-oauth2'; -import { Logger } from 'winston'; -import { TokenIssuer } from '../../identity'; import { - EnvironmentHandler, - EnvironmentHandlers, -} from '../../lib/EnvironmentHandler'; -import { OAuthProvider } from '../../lib/OAuthProvider'; + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; import { executeFetchUserProfileStrategy, executeFrameHandlerStrategy, executeRedirectStrategy, executeRefreshTokenStrategy, makeProfileInfo, -} from '../../lib/PassportStrategyHelper'; -import { - AuthProviderConfig, - EnvironmentProviderConfig, - GenericOAuth2ProviderConfig, - GenericOAuth2ProviderOptions, - OAuthProviderHandlers, - OAuthResponse, PassportDoneCallback, - RedirectInfo, -} from '../types'; +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; type PrivateInfo = { refreshToken: string; }; -export class OAuth2AuthProvider implements OAuthProviderHandlers { +export type OAuth2AuthProviderOptions = OAuthProviderOptions & { + authorizationUrl: string; + tokenUrl: string; +}; + +export class OAuth2AuthProvider implements OAuthHandlers { private readonly _strategy: OAuth2Strategy; - constructor(options: GenericOAuth2ProviderOptions) { + constructor(options: OAuth2AuthProviderOptions) { this._strategy = new OAuth2Strategy( - { ...options, passReqToCallback: false as true }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + authorizationURL: options.authorizationUrl, + tokenURL: options.tokenUrl, + passReqToCallback: false as true, + }, ( accessToken: any, refreshToken: any, @@ -60,6 +67,7 @@ export class OAuth2AuthProvider implements OAuthProviderHandlers { done: PassportDoneCallback, ) => { const profile = makeProfileInfo(rawProfile, params.id_token); + done( undefined, { @@ -79,16 +87,13 @@ export class OAuth2AuthProvider implements OAuthProviderHandlers { ); } - async start( - req: express.Request, - options: Record, - ): Promise { - const providerOptions = { - ...options, + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { accessType: 'offline', prompt: 'consent', - }; - return await executeRedirectStrategy(req, this._strategy, providerOptions); + scope: req.scope, + state: encodeState(req.state), + }); } async handler( @@ -105,12 +110,17 @@ export class OAuth2AuthProvider implements OAuthProviderHandlers { }; } - async refresh(refreshToken: string, scope: string): Promise { - const { accessToken, params } = await executeRefreshTokenStrategy( + async refresh(req: OAuthRefreshRequest): Promise { + const refreshTokenResponse = await executeRefreshTokenStrategy( this._strategy, - refreshToken, - scope, + req.refreshToken, + req.scope, ); + const { + accessToken, + params, + refreshToken: updatedRefreshToken, + } = refreshTokenResponse; const profile = await executeFetchUserProfileStrategy( this._strategy, @@ -121,6 +131,7 @@ export class OAuth2AuthProvider implements OAuthProviderHandlers { return this.populateIdentity({ providerInfo: { accessToken, + refreshToken: updatedRefreshToken, idToken: params.id_token, expiresInSeconds: params.expires_in, scope: params.scope, @@ -139,60 +150,36 @@ export class OAuth2AuthProvider implements OAuthProviderHandlers { if (!profile.email) { throw new Error('Profile does not contain a profile'); } - const id = profile.email.split('@')[0]; return { ...response, backstageIdentity: { id } }; } } -export function createOAuth2Provider( - { baseUrl }: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const providerId = 'oauth2'; - const envProviders: EnvironmentHandlers = {}; +export const createOAuth2Provider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'oauth2'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + const authorizationUrl = envConfig.getString('authorizationUrl'); + const tokenUrl = envConfig.getString('tokenUrl'); - for (const [env, envConfig] of Object.entries(providerConfig)) { - const config = (envConfig as unknown) as GenericOAuth2ProviderConfig; - const { secure, appOrigin } = config; - const opts = { - clientID: config.clientId, - clientSecret: config.clientSecret, - callbackURL: `${baseUrl}/${providerId}/handler/frame?env=${env}`, - authorizationURL: config.authorizationURL, - tokenURL: config.tokenURL, - }; + const provider = new OAuth2AuthProvider({ + clientId, + clientSecret, + callbackUrl, + authorizationUrl, + tokenUrl, + }); - if ( - !opts.clientID || - !opts.clientSecret || - !opts.authorizationURL || - !opts.tokenURL - ) { - if (process.env.NODE_ENV !== 'development') { - throw new Error( - 'Failed to initialize OAuth2 auth provider, set AUTH_OAUTH2_CLIENT_ID, AUTH_OAUTH2_CLIENT_SECRET, AUTH_OAUTH2_AUTH_URL, and AUTH_OAUTH2_TOKEN_URL env vars', - ); - } - - logger.warn( - 'OAuth2 auth provider disabled, set AUTH_OAUTH2_CLIENT_ID, AUTH_OAUTH2_CLIENT_SECRET, AUTH_OAUTH2_AUTH_URL, and AUTH_OAUTH2_TOKEN_URL env vars to enable', - ); - continue; - } - - envProviders[env] = new OAuthProvider(new OAuth2AuthProvider(opts), { + return OAuthAdapter.fromConfig(globalConfig, provider, { disableRefresh: false, providerId, - secure, - baseUrl, - appOrigin, tokenIssuer, }); - } - - return new EnvironmentHandler(providerId, envProviders); -} + }); diff --git a/plugins/auth-backend/src/providers/okta/index.ts b/plugins/auth-backend/src/providers/okta/index.ts index bc32601ac2..05cc398f43 100644 --- a/plugins/auth-backend/src/providers/okta/index.ts +++ b/plugins/auth-backend/src/providers/okta/index.ts @@ -13,4 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { createOktaProvider } from './provider'; +export { createOktaProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index 01d7684e97..09597696ba 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -14,7 +14,16 @@ * limitations under the License. */ import express from 'express'; -import { OAuthProvider } from '../../lib/OAuthProvider'; +import { + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; import { Strategy as OktaStrategy } from 'passport-okta-oauth'; import passport from 'passport'; import { @@ -23,30 +32,20 @@ import { executeRefreshTokenStrategy, makeProfileInfo, executeFetchUserProfileStrategy, -} from '../../lib/PassportStrategyHelper'; -import { - OAuthProviderHandlers, - RedirectInfo, - AuthProviderConfig, - EnvironmentProviderConfig, - OAuthProviderOptions, - OAuthProviderConfig, - OAuthResponse, PassportDoneCallback, -} from '../types'; -import { - EnvironmentHandler, - EnvironmentHandlers, -} from '../../lib/EnvironmentHandler'; -import { Logger } from 'winston'; +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory } from '../types'; import { StateStore } from 'passport-oauth2'; -import { TokenIssuer } from '../../identity'; type PrivateInfo = { refreshToken: string; }; -export class OktaAuthProvider implements OAuthProviderHandlers { +export type OktaAuthProviderOptions = OAuthProviderOptions & { + audience: string; +}; + +export class OktaAuthProvider implements OAuthHandlers { private readonly _strategy: any; /** @@ -66,11 +65,14 @@ export class OktaAuthProvider implements OAuthProviderHandlers { }, }; - constructor(options: OAuthProviderOptions) { + constructor(options: OktaAuthProviderOptions) { this._strategy = new OktaStrategy( { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + audience: options.audience, passReqToCallback: false as true, - ...options, store: this._store, response_type: 'code', }, @@ -102,16 +104,13 @@ export class OktaAuthProvider implements OAuthProviderHandlers { ); } - async start( - req: express.Request, - options: Record, - ): Promise { - const providerOptions = { - ...options, + async start(req: OAuthStartRequest): Promise { + return await executeRedirectStrategy(req, this._strategy, { accessType: 'offline', prompt: 'consent', - }; - return await executeRedirectStrategy(req, this._strategy, providerOptions); + scope: req.scope, + state: encodeState(req.state), + }); } async handler( @@ -128,11 +127,11 @@ export class OktaAuthProvider implements OAuthProviderHandlers { }; } - async refresh(refreshToken: string, scope: string): Promise { + async refresh(req: OAuthRefreshRequest): Promise { const { accessToken, params } = await executeRefreshTokenStrategy( this._strategy, - refreshToken, - scope, + req.refreshToken, + req.scope, ); const profile = await executeFetchUserProfileStrategy( @@ -168,47 +167,28 @@ export class OktaAuthProvider implements OAuthProviderHandlers { } } -export function createOktaProvider( - { baseUrl }: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const providerId = 'okta'; - const envProviders: EnvironmentHandlers = {}; +export const createOktaProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'okta'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const audience = envConfig.getString('audience'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - for (const [env, envConfig] of Object.entries(providerConfig)) { - const config = (envConfig as unknown) as OAuthProviderConfig; - const { secure, appOrigin } = config; - const opts = { - audience: config.audience, - clientID: config.clientId, - clientSecret: config.clientSecret, - callbackURL: `${baseUrl}/${providerId}/handler/frame?env=${env}`, - }; + const provider = new OktaAuthProvider({ + audience, + clientId, + clientSecret, + callbackUrl, + }); - if (!opts.clientID || !opts.clientSecret || !opts.audience) { - if (process.env.NODE_ENV !== 'development') { - throw new Error( - 'Failed to initialize Okta auth provider, set AUTH_OKTA_CLIENT_ID, AUTH_OKTA_CLIENT_SECRET, and AUTH_OKTA_AUDIENCE env vars', - ); - } - - logger.warn( - 'Okta auth provider disabled, set AUTH_OKTA_CLIENT_ID, AUTH_OKTA_CLIENT_SECRET, and AUTH_OKTA_AUDIENCE env vars to enable', - ); - continue; - } - - envProviders[env] = new OAuthProvider(new OktaAuthProvider(opts), { + return OAuthAdapter.fromConfig(globalConfig, provider, { disableRefresh: false, providerId, - secure, - baseUrl, - appOrigin, tokenIssuer, }); - } - - return new EnvironmentHandler(providerId, envProviders); -} + }); diff --git a/plugins/auth-backend/src/providers/okta/types.d.ts b/plugins/auth-backend/src/providers/okta/types.d.ts index bed6d24043..6b49d99817 100644 --- a/plugins/auth-backend/src/providers/okta/types.d.ts +++ b/plugins/auth-backend/src/providers/okta/types.d.ts @@ -14,9 +14,7 @@ * limitations under the License. */ declare module 'passport-okta-oauth' { - export class Strategy { - constructor(options: any, verify: any) + constructor(options: any, verify: any); } } - \ No newline at end of file diff --git a/plugins/auth-backend/src/providers/saml/provider.ts b/plugins/auth-backend/src/providers/saml/provider.ts index 40c80de4e4..2bd8ed0bf3 100644 --- a/plugins/auth-backend/src/providers/saml/provider.ts +++ b/plugins/auth-backend/src/providers/saml/provider.ts @@ -23,21 +23,14 @@ import { import { executeFrameHandlerStrategy, executeRedirectStrategy, -} from '../../lib/PassportStrategyHelper'; -import { - AuthProviderConfig, - AuthProviderRouteHandlers, - EnvironmentProviderConfig, - SAMLProviderConfig, PassportDoneCallback, - ProfileInfo, -} from '../types'; -import { postMessageResponse } from '../../lib/OAuthProvider'; +} from '../../lib/passport'; import { - EnvironmentHandlers, - EnvironmentHandler, -} from '../../lib/EnvironmentHandler'; -import { Logger } from 'winston'; + AuthProviderRouteHandlers, + ProfileInfo, + AuthProviderFactory, +} from '../types'; +import { postMessageResponse } from '../../lib/flow'; import { TokenIssuer } from '../../identity'; type SamlInfo = { @@ -111,6 +104,10 @@ export class SamlAuthProvider implements AuthProviderRouteHandlers { async logout(_req: express.Request, res: express.Response): Promise { res.send('noop'); } + + identifyEnv(): string | undefined { + return undefined; + } } type SAMLProviderOptions = { @@ -120,32 +117,18 @@ type SAMLProviderOptions = { tokenIssuer: TokenIssuer; }; -export function createSamlProvider( - _authProviderConfig: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - tokenIssuer: TokenIssuer, -) { - const envProviders: EnvironmentHandlers = {}; +export const createSamlProvider: AuthProviderFactory = ({ + config, + tokenIssuer, +}) => { + const entryPoint = config.getString('entryPoint'); + const issuer = config.getString('issuer'); + const opts = { + entryPoint, + issuer, + path: '/auth/saml/handler/frame', + tokenIssuer, + }; - for (const [env, envConfig] of Object.entries(providerConfig)) { - const config = (envConfig as unknown) as SAMLProviderConfig; - const opts = { - entryPoint: config.entryPoint, - issuer: config.issuer, - path: '/auth/saml/handler/frame', - tokenIssuer, - }; - - if (!opts.entryPoint || !opts.issuer) { - logger.warn( - 'SAML auth provider disabled, set entryPoint and entryPoint in saml auth config to enable', - ); - continue; - } - - envProviders[env] = new SamlAuthProvider(opts); - } - - return new EnvironmentHandler('saml', envProviders); -} + return new SamlAuthProvider(opts); +}; diff --git a/plugins/auth-backend/src/providers/types.ts b/plugins/auth-backend/src/providers/types.ts index de4e076919..5c05b02739 100644 --- a/plugins/auth-backend/src/providers/types.ts +++ b/plugins/auth-backend/src/providers/types.ts @@ -17,72 +17,7 @@ import express from 'express'; import { Logger } from 'winston'; import { TokenIssuer } from '../identity'; - -export type OAuthProviderOptions = { - /** - * Client ID of the auth provider. - */ - clientID: string; - /** - * Client Secret of the auth provider. - */ - clientSecret: string; - /** - * Callback URL to be passed to the auth provider to redirect to after the user signs in. - */ - callbackURL: string; -}; - -export type GenericOAuth2ProviderOptions = OAuthProviderOptions & { - authorizationURL: string; - tokenURL: string; -}; - -export type OAuthProviderConfig = { - /** - * Cookies can be marked with a secure flag to send cookies only when the request - * is over an encrypted channel (HTTPS). - * - * For development environment we don't mark the cookie as secure since we serve - * localhost over HTTP. - */ - secure: boolean; - /** - * The protocol://domain[:port] where the app (frontend) is hosted. This is used to post messages back - * to the window that initiates an auth request. - */ - appOrigin: string; - /** - * Client ID of the auth provider. - */ - clientId: string; - /** - * Client Secret of the auth provider. - */ - clientSecret: string; - /** - * The location of the OAuth Authorization Server - */ - audience?: string; -}; - -export type GenericOAuth2ProviderConfig = OAuthProviderConfig & { - authorizationURL: string; - tokenURL: string; -}; - -export type EnvironmentProviderConfig = { - /** - * key, values are environment names and OAuthProviderConfigs - * - * For e.g - * { - * development: DevelopmentOAuthProviderConfig - * production: ProductionOAuthProviderConfig - * } - */ - [key: string]: OAuthProviderConfig; -}; +import { Config } from '@backstage/config'; export type AuthProviderConfig = { /** @@ -90,50 +25,23 @@ export type AuthProviderConfig = { * callbackURL to redirect to once the user signs in to the auth provider. */ baseUrl: string; + + /** + * The base URL of the app as provided by app.baseUrl + */ + appUrl: string; }; -/** - * Any OAuth provider needs to implement this interface which has provider specific - * handlers for different methods to perform authentication, get access tokens, - * refresh tokens and perform sign out. - */ -export interface OAuthProviderHandlers { +export type RedirectInfo = { /** - * This method initiates a sign in request with an auth provider. - * @param {express.Request} req - * @param options + * URL to redirect to */ - start( - req: express.Request, - options: Record, - ): Promise; - + url: string; /** - * Handles the redirect from the auth provider when the user has signed in. - * @param {express.Request} req + * Status code to use for the redirect */ - handler( - req: express.Request, - ): Promise<{ - response: AuthResponse; - refreshToken?: string; - }>; - - /** - * (Optional) Given a refresh token and scope fetches a new access token from the auth provider. - * @param {string} refreshToken - * @param {string} scope - */ - refresh?( - refreshToken: string, - scope: string, - ): Promise>; - - /** - * (Optional) Sign out of the auth provider. - */ - logout?(): Promise; -} + status?: number; +}; /** * Any Auth provider needs to implement this interface which handles the routes in the @@ -202,11 +110,15 @@ export interface AuthProviderRouteHandlers { logout?(req: express.Request, res: express.Response): Promise; } +export type AuthProviderFactoryOptions = { + globalConfig: AuthProviderConfig; + config: Config; + logger: Logger; + tokenIssuer: TokenIssuer; +}; + export type AuthProviderFactory = ( - globalConfig: AuthProviderConfig, - providerConfig: EnvironmentProviderConfig, - logger: Logger, - issuer: TokenIssuer, + options: AuthProviderFactoryOptions, ) => AuthProviderRouteHandlers; export type AuthResponse = { @@ -215,8 +127,6 @@ export type AuthResponse = { backstageIdentity?: BackstageIdentity; }; -export type OAuthResponse = AuthResponse; - export type BackstageIdentity = { /** * The backstage user ID. @@ -229,63 +139,6 @@ export type BackstageIdentity = { idToken?: string; }; -export type OAuthProviderInfo = { - /** - * An access token issued for the signed in user. - */ - accessToken: string; - /** - * (Optional) Id token issued for the signed in user. - */ - idToken?: string; - /** - * Expiry of the access token in seconds. - */ - expiresInSeconds?: number; - /** - * Scopes granted for the access token. - */ - scope: string; -}; - -export type OAuthPrivateInfo = { - /** - * A refresh token issued for the signed in user. - */ - refreshToken: string; -}; - -/** - * Payload sent as a post message after the auth request is complete. - * If successful then has a valid payload with Auth information else contains an error. - */ -export type WebMessageResponse = - | { - type: 'authorization_response'; - response: AuthResponse; - } - | { - type: 'authorization_response'; - error: Error; - }; - -export type PassportDoneCallback = ( - err?: Error, - response?: Res, - privateInfo?: Private, -) => void; - -export type RedirectInfo = { - /** - * URL to redirect to - */ - url: string; - /** - * Status code to use for the redirect - */ - status?: number; -}; - /** * Used to display login information to user, i.e. sidebar popup. * @@ -307,24 +160,3 @@ export type ProfileInfo = { */ picture?: string; }; - -export type RefreshTokenResponse = { - /** - * An access token issued for the signed in user. - */ - accessToken: string; - params: any; -}; - -export type ProviderStrategy = { - userProfile(accessToken: string, callback: Function): void; -}; - -export type SAMLProviderConfig = { - entryPoint: string; - issuer: string; -}; - -export type SAMLEnvironmentProviderConfig = { - [key: string]: SAMLProviderConfig; -}; diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 6b296cba2d..19a74d47c5 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -17,12 +17,12 @@ import express from 'express'; import Router from 'express-promise-router'; import cookieParser from 'cookie-parser'; -import bodyParser from 'body-parser'; import Knex from 'knex'; import { Logger } from 'winston'; import { createAuthProviderRouter } from '../providers'; import { Config } from '@backstage/config'; import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity'; +import { NotFoundError } from '@backstage/backend-common'; export interface RouterOptions { logger: Logger; @@ -36,8 +36,7 @@ export async function createRouter( const router = Router(); const logger = options.logger.child({ plugin: 'auth' }); - const appUrl = new URL(options.config.getString('app.baseUrl')); - const appOrigin = appUrl.origin; + const appUrl = options.config.getString('app.baseUrl'); const backendUrl = options.config.getString('backend.baseUrl'); const authUrl = `${backendUrl}/auth`; @@ -54,84 +53,32 @@ export async function createRouter( }); router.use(cookieParser()); - router.use(bodyParser.urlencoded({ extended: false })); - router.use(bodyParser.json()); + router.use(express.urlencoded({ extended: false })); + router.use(express.json()); - const config = { - backend: { - baseUrl: backendUrl, - }, - auth: { - providers: { - google: { - development: { - appOrigin, - secure: false, - clientId: process.env.AUTH_GOOGLE_CLIENT_ID!, - clientSecret: process.env.AUTH_GOOGLE_CLIENT_SECRET!, - }, - }, - github: { - development: { - appOrigin, - secure: false, - clientId: process.env.AUTH_GITHUB_CLIENT_ID!, - clientSecret: process.env.AUTH_GITHUB_CLIENT_SECRET!, - }, - }, - gitlab: { - development: { - appOrigin, - secure: false, - clientId: process.env.AUTH_GITLAB_CLIENT_ID!, - clientSecret: process.env.AUTH_GITLAB_CLIENT_SECRET!, - audience: process.env.GITLAB_BASE_URL! || 'https://gitlab.com', - }, - }, - saml: { - development: { - entryPoint: 'http://localhost:7001/', - issuer: 'passport-saml', - }, - }, - okta: { - development: { - appOrigin, - secure: false, - clientId: process.env.AUTH_OKTA_CLIENT_ID!, - clientSecret: process.env.AUTH_OKTA_CLIENT_SECRET!, - audience: process.env.AUTH_OKTA_AUDIENCE, - }, - }, - oauth2: { - development: { - appOrigin, - secure: false, - clientId: process.env.AUTH_OAUTH2_CLIENT_ID!, - clientSecret: process.env.AUTH_OAUTH2_CLIENT_SECRET!, - authorizationURL: process.env.AUTH_OAUTH2_AUTH_URL!, - tokenURL: process.env.AUTH_OAUTH2_TOKEN_URL!, - }, - }, - }, - }, - }; + const providersConfig = options.config.getConfig('auth.providers'); + const providers = providersConfig.keys(); - const providerConfigs = config.auth.providers; - - for (const [providerId, providerConfig] of Object.entries(providerConfigs)) { + for (const providerId of providers) { logger.info(`Configuring provider, ${providerId}`); try { + const providerConfig = providersConfig.getConfig(providerId); const providerRouter = createAuthProviderRouter( providerId, - { baseUrl: authUrl }, + { baseUrl: authUrl, appUrl }, providerConfig, logger, tokenIssuer, ); router.use(`/${providerId}`, providerRouter); } catch (e) { - logger.error(e.message); + if (process.env.NODE_ENV !== 'development') { + throw new Error( + `Failed to initialize ${providerId} auth provider, ${e.message}`, + ); + } + + logger.warn(`Skipping ${providerId} auth provider, ${e.message}`); } } @@ -142,5 +89,10 @@ export async function createRouter( }), ); + router.use('/:provider/', req => { + const { provider } = req.params; + throw new NotFoundError(`No auth provider registered for '${provider}'`); + }); + return router; } diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts index d2372a7e91..dc91c92631 100644 --- a/plugins/auth-backend/src/service/standaloneServer.ts +++ b/plugins/auth-backend/src/service/standaloneServer.ts @@ -18,9 +18,12 @@ import Knex from 'knex'; import { Server } from 'http'; import { Logger } from 'winston'; import { ConfigReader } from '@backstage/config'; -import { loadConfig } from '@backstage/config-loader'; import { createRouter } from './router'; -import { createServiceBuilder, useHotMemoize } from '@backstage/backend-common'; +import { + createServiceBuilder, + useHotMemoize, + loadBackendConfig, +} from '@backstage/backend-common'; export interface ServerOptions { logger: Logger; @@ -30,7 +33,7 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'auth-backend' }); - const config = ConfigReader.fromConfigs(await loadConfig()); + const config = ConfigReader.fromConfigs(await loadBackendConfig()); const database = useHotMemoize(module, () => { const knex = Knex({ diff --git a/plugins/catalog-backend/README.md b/plugins/catalog-backend/README.md index 246cd83d56..1b4a653f4a 100644 --- a/plugins/catalog-backend/README.md +++ b/plugins/catalog-backend/README.md @@ -21,14 +21,11 @@ To evaluate the catalog and have a greater amount of functionality available, in # in one terminal window, run this from from the very root of the Backstage project cd packages/backend yarn start - -# open another terminal window, and run the following from the very root of the Backstage project -yarn lerna run mock-data ``` -This will launch the full example backend and populate its catalog with some mock entities. +This will launch the full example backend, populated some example entities. ## Links -- (Frontend part of the plugin)[https://github.com/spotify/backstage/tree/master/plugins/catalog] -- (The Backstage homepage)[https://backstage.io] +- [Frontend part of the plugin](https://github.com/spotify/backstage/tree/master/plugins/catalog) +- [The Backstage homepage](https://backstage.io) diff --git a/plugins/catalog-backend/migrations/20200721115244_location_update_log_latest_deduplicate.js b/plugins/catalog-backend/migrations/20200721115244_location_update_log_latest_deduplicate.js index df43a22c56..9046f85da0 100644 --- a/plugins/catalog-backend/migrations/20200721115244_location_update_log_latest_deduplicate.js +++ b/plugins/catalog-backend/migrations/20200721115244_location_update_log_latest_deduplicate.js @@ -25,7 +25,7 @@ exports.up = function up(knex) { ) t2 ON t1.location_id = t2.location_id AND t1.created_at = t2.MAXDATE - GROUP BY t1.location_id + GROUP BY t1.location_id, t1.id ORDER BY created_at DESC; `); }; diff --git a/plugins/catalog-backend/migrations/20200805163904_location_update_log_duplication_fix.js b/plugins/catalog-backend/migrations/20200805163904_location_update_log_duplication_fix.js new file mode 100644 index 0000000000..05c1658641 --- /dev/null +++ b/plugins/catalog-backend/migrations/20200805163904_location_update_log_duplication_fix.js @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = function up(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = function down(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js new file mode 100644 index 0000000000..6e02975f92 --- /dev/null +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + try { + await knex.schema.alterTable('entities_search', table => { + table.text('value').nullable().alter(); + }); + } catch (e) { + // Sqlite does not support alter column. + } +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + try { + await knex.schema.alterTable('entities_search', table => { + table.string('value').nullable().alter(); + }); + } catch (e) { + // Sqlite does not support alter column. + } +}; diff --git a/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js new file mode 100644 index 0000000000..379928493d --- /dev/null +++ b/plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + // Adds a single 'bootstrap' location that can be used to trigger work in processors. + // This is primarily here to fulfill foreign key constraints. + await knex('locations').insert({ + id: require('uuid').v4(), + type: 'bootstrap', + target: 'bootstrap', + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + await knex('locations') + .where({ + type: 'bootstrap', + target: 'bootstrap', + }) + .del(); +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9c7a18087d..f443ff362f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.1.1-alpha.15", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -17,16 +17,17 @@ "test": "backstage-cli test", "prepack": "backstage-cli prepack", "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean", - "mock-data": "./scripts/mock-data.sh" + "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.15", - "@backstage/catalog-model": "^0.1.1-alpha.15", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", + "git-url-parse": "^11.2.0", "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", @@ -39,13 +40,15 @@ "yup": "^0.29.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.15", + "@backstage/cli": "^0.1.1-alpha.22", + "@types/git-url-parse": "^9.0.0", "@types/lodash": "^4.14.151", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", + "msw": "^0.20.5", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/catalog-backend/scripts/mock-data.sh b/plugins/catalog-backend/scripts/mock-data.sh deleted file mode 100755 index 58e2efbe21..0000000000 --- a/plugins/catalog-backend/scripts/mock-data.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -for URL in \ - 'artist-lookup-component.yaml' \ - 'playback-order-component.yaml' \ - 'podcast-api-component.yaml' \ - 'queue-proxy-component.yaml' \ - 'searcher-component.yaml' \ - 'playback-lib-component.yaml' \ - 'www-artist-component.yaml' \ - 'shuffle-api-component.yaml' \ -; do \ - curl \ - --location \ - --request POST 'localhost:7000/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"github\", \"target\": \"https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/${URL}\"}" - echo -done - -curl \ - --location \ - --request POST 'localhost:7000/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"github\", \"target\": \"https://github.com/benjdlambert/cookiecutter-golang/blob/master/template.yaml\"}" -echo - diff --git a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts new file mode 100644 index 0000000000..bcd1248a66 --- /dev/null +++ b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import { CoalescedEntitiesCatalog } from './CoalescedEntitiesCatalog'; +import { EntitiesCatalog } from './types'; + +describe('CoalescedEntitiesCatalog', () => { + const e1: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n1' }, + }; + + const e2: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n2' }, + }; + + const c1: jest.Mocked = { + entities: jest.fn(), + entityByUid: jest.fn(), + entityByName: jest.fn(), + addOrUpdateEntity: jest.fn(), + removeEntityByUid: jest.fn(), + }; + + const c2: jest.Mocked = { + entities: jest.fn(), + entityByUid: jest.fn(), + entityByName: jest.fn(), + addOrUpdateEntity: jest.fn(), + removeEntityByUid: jest.fn(), + }; + + const mockLogger = { + warn: jest.fn(), + }; + const logger = (mockLogger as unknown) as Logger; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('entities', () => { + it('flattens results from multiple sources', async () => { + c1.entities.mockResolvedValueOnce([e1]); + c2.entities.mockResolvedValueOnce([e2]); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entities()).resolves.toEqual( + expect.arrayContaining([e1, e2]), + ); + expect(c1.entities).toBeCalledTimes(1); + expect(c2.entities).toBeCalledTimes(1); + }); + + it('logs an error if any source throws', async () => { + c1.entities.mockResolvedValueOnce([e1]); + c2.entities.mockRejectedValueOnce(new Error('boo')); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entities()).resolves.toEqual([e1]); + expect(c1.entities).toBeCalledTimes(1); + expect(c2.entities).toBeCalledTimes(1); + expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/)); + }); + }); + + describe('entityByUid', () => { + it('returns the first non-undefined result', async () => { + c1.entityByUid.mockResolvedValueOnce(undefined); + c2.entityByUid.mockResolvedValueOnce(e2); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entityByUid('e2')).resolves.toBe(e2); + expect(c1.entityByUid).toBeCalledTimes(1); + expect(c2.entityByUid).toBeCalledTimes(1); + }); + + it('returns undefined if all results were undefined', async () => { + c1.entityByUid.mockResolvedValueOnce(undefined); + c2.entityByUid.mockResolvedValueOnce(undefined); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entityByUid('e2')).resolves.toBeUndefined(); + expect(c1.entityByUid).toBeCalledTimes(1); + expect(c2.entityByUid).toBeCalledTimes(1); + }); + + it('logs an error if any source throws', async () => { + c1.entityByUid.mockResolvedValueOnce(e1); + c2.entityByUid.mockRejectedValueOnce(new Error('boo')); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entityByUid('e2')).resolves.toBe(e1); + expect(c1.entityByUid).toBeCalledTimes(1); + expect(c2.entityByUid).toBeCalledTimes(1); + expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/)); + }); + }); + + describe('entityByName', () => { + it('returns the first non-undefined result', async () => { + c1.entityByName.mockResolvedValueOnce(undefined); + c2.entityByName.mockResolvedValueOnce(e2); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entityByName('k', undefined, 'n2')).resolves.toBe( + e2, + ); + expect(c1.entityByName).toBeCalledTimes(1); + expect(c2.entityByName).toBeCalledTimes(1); + }); + + it('returns undefined if all results were undefined', async () => { + c1.entityByName.mockResolvedValueOnce(undefined); + c2.entityByName.mockResolvedValueOnce(undefined); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect( + catalog.entityByName('k', undefined, 'n2'), + ).resolves.toBeUndefined(); + expect(c1.entityByName).toBeCalledTimes(1); + expect(c2.entityByName).toBeCalledTimes(1); + }); + + it('logs an error if any source throws', async () => { + c1.entityByName.mockResolvedValueOnce(e1); + c2.entityByName.mockRejectedValueOnce(new Error('boo')); + const catalog = new CoalescedEntitiesCatalog([c1, c2], logger); + await expect(catalog.entityByName('k', undefined, 'n2')).resolves.toBe( + e1, + ); + expect(c1.entityByName).toBeCalledTimes(1); + expect(c2.entityByName).toBeCalledTimes(1); + expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/)); + }); + }); +}); diff --git a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts new file mode 100644 index 0000000000..5a0922495e --- /dev/null +++ b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts @@ -0,0 +1,100 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import { EntityFilters } from '../database'; +import { EntitiesCatalog } from './types'; + +/** + * A simple coalescing catalog wrapper, that acts as a front for collecting + * catalog data from multiple sources. + * + * One possible usage could be to have this as a front to both a + * DatabaseEntitiesCatalog that holds Component kinds, and another company- + * specific catalog that is a thin wrapper on top of LDAP that supplies Group + * and User entities. That way you'll get a coherent view of two very different + * entity sources. + * + * This is mainly meant as a functional example, and you may want to provide + * your own more specialized collector if you have this distinct need. This + * one does not support adding/updating entities through the API for example. + * A more competent implementation may direct the writes to different catalogs + * based on entity kind or similar. + */ +export class CoalescedEntitiesCatalog implements EntitiesCatalog { + private inner: EntitiesCatalog[]; + private logger: Logger; + + constructor(inner: EntitiesCatalog[], logger: Logger) { + this.inner = inner; + this.logger = logger; + } + + async entities(filters?: EntityFilters): Promise { + const ops = this.inner.map(async catalog => { + try { + return await catalog.entities(filters); + } catch (e) { + this.logger.warn(`Inner entities call failed, ${e}`); + return []; + } + }); + + const results = await Promise.all(ops); + return results.flat(); + } + + async entityByUid(uid: string): Promise { + const ops = this.inner.map(async catalog => { + try { + return await catalog.entityByUid(uid); + } catch (e) { + this.logger.warn(`Inner entityByUid call failed, ${e}`); + return undefined; + } + }); + + const results = await Promise.all(ops); + return results.find(Boolean); + } + + async entityByName( + kind: string, + namespace: string | undefined, + name: string, + ): Promise { + const ops = this.inner.map(async catalog => { + try { + return await catalog.entityByName(kind, namespace, name); + } catch (e) { + this.logger.warn(`Inner entityByName call failed, ${e}`); + return undefined; + } + }); + + const results = await Promise.all(ops); + return results.find(Boolean); + } + + addOrUpdateEntity(): Promise { + throw new Error('Method not implemented.'); + } + + removeEntityByUid(): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 8c7897fa2c..958e864a8e 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -17,6 +17,12 @@ import { DatabaseManager } from '../database'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +const bootstrapLocation = { + id: expect.any(String), + type: 'bootstrap', + target: 'bootstrap', +}; + describe('DatabaseLocationsCatalog', () => { let catalog: DatabaseLocationsCatalog; @@ -35,8 +41,41 @@ describe('DatabaseLocationsCatalog', () => { await expect( catalog.location('dd12620d-0436-422f-93bd-929aa0788123'), ).resolves.toEqual(expect.objectContaining({ data: location })); - await expect(catalog.locations()).resolves.toEqual([ - expect.objectContaining({ data: location }), - ]); + await expect(catalog.locations()).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ data: location }), + expect.objectContaining({ data: bootstrapLocation }), + ]), + ); + }); + + it('does not return duplicates of rows because of logs', async () => { + const location1 = { + id: 'dd12620d-0436-422f-93bd-929aa0788123', + type: 'valid_type', + target: 'valid_target1', + }; + const location2 = { + id: '1a89c479-1a33-4f27-8927-6090ba488c42', + type: 'valid_type', + target: 'valid_target2', + }; + await expect(catalog.addLocation(location1)).resolves.toEqual(location1); + await expect(catalog.addLocation(location2)).resolves.toEqual(location2); + await expect( + catalog.logUpdateSuccess(location1.id), + ).resolves.toBeUndefined(); + await expect( + catalog.logUpdateSuccess(location1.id), + ).resolves.toBeUndefined(); + const locations = await catalog.locations(); + expect(locations.length).toBe(3); + expect(locations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ data: location1 }), + expect.objectContaining({ data: location2 }), + expect.objectContaining({ data: bootstrapLocation }), + ]), + ); }); }); diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index 21dde4ca74..b86b4ef959 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -24,6 +24,15 @@ import type { DbLocationsRowWithStatus, } from './types'; +const bootstrapLocation = { + id: expect.any(String), + type: 'bootstrap', + target: 'bootstrap', + message: null, + status: null, + timestamp: null, +}; + describe('CommonDatabase', () => { let db: Database; let entityRequest: DbEntityRequest; @@ -85,8 +94,12 @@ describe('CommonDatabase', () => { await db.addLocation(input); const locations = await db.locations(); - expect(locations).toEqual([output]); - const location = await db.location(locations[0].id); + expect(locations).toEqual( + expect.arrayContaining([output, bootstrapLocation]), + ); + const location = await db.location( + locations.find(l => l.type !== 'bootstrap')!.id, + ); expect(location).toEqual(output); // If we add 2 new update log events, @@ -105,20 +118,21 @@ describe('CommonDatabase', () => { DatabaseLocationUpdateLogStatus.FAIL, ); - expect(await db.locations()).toEqual([ - { - ...output, - status: DatabaseLocationUpdateLogStatus.FAIL, - timestamp: expect.any(String), - }, - ]); - - await db.transaction(tx => db.removeLocation(tx, locations[0].id)); - - await expect(db.locations()).resolves.toEqual([]); - await expect(db.location(locations[0].id)).rejects.toThrow( - /Found no location/, + await expect(db.locations()).resolves.toEqual( + expect.arrayContaining([ + bootstrapLocation, + { + ...output, + status: DatabaseLocationUpdateLogStatus.FAIL, + timestamp: expect.any(String), + }, + ]), ); + + await db.transaction(tx => db.removeLocation(tx, location.id)); + + await expect(db.locations()).resolves.toEqual([bootstrapLocation]); + await expect(db.location(location.id)).rejects.toThrow(/Found no location/); }); describe('addEntity', () => { diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index c860f52d1e..4ad5362fb6 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -29,7 +29,6 @@ import { } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; -import { v4 as uuidv4 } from 'uuid'; import type { Logger } from 'winston'; import { buildEntitySearch } from './search'; import type { @@ -344,10 +343,16 @@ export class CommonDatabase implements Database { entityName?: string, message?: string, ): Promise { - return this.database( + // Remove log entries older than a day + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - 1); + await this.database('location_update_log') + .where('created_at', '<', cutoff.toISOString()) + .del(); + + await this.database( 'location_update_log', ).insert({ - id: uuidv4(), status, location_id: locationId, entity_name: entityName, diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 28751f4830..ce91cc737f 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,17 +14,16 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; import { makeValidator } from '@backstage/catalog-model'; import Knex from 'knex'; -import path from 'path'; import { Logger } from 'winston'; import { CommonDatabase } from './CommonDatabase'; import { Database } from './types'; -const migrationsDir = path.resolve( - require.resolve('@backstage/plugin-catalog-backend/package.json'), - '../migrations', +const migrationsDir = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrations', ); export type CreateDatabaseOptions = { diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts new file mode 100644 index 0000000000..bb5c025c28 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -0,0 +1,217 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec, Entity } from '@backstage/catalog-model'; +import { CatalogRulesEnforcer } from './CatalogRules'; +import { ConfigReader } from '@backstage/config'; + +const entity = { + user: { + kind: 'User', + } as Entity, + group: { + kind: 'Group', + } as Entity, + component: { + kind: 'component', + } as Entity, + location: { + kind: 'Location', + } as Entity, +}; + +const location: Record = { + x: { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + }, + y: { + type: 'github', + target: 'https://github.com/a/b/blob/master/y.yaml', + }, + z: { + type: 'file', + target: '/root/z.yaml', + }, +}; + +describe('CatalogRulesEnforcer', () => { + it('should deny by default', () => { + const enforcer = new CatalogRulesEnforcer([]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + + it('should deny all', () => { + const enforcer = new CatalogRulesEnforcer([{ allow: [] }]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + + it('should allow all', () => { + const enforcer = new CatalogRulesEnforcer([ + { + allow: ['User', 'Group', 'Component', 'Location'].map(kind => ({ + kind, + })), + }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups from github', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should allow groups from files', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should not be sensitive to kind case', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'group' }] }, + { allow: [{ kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + describe('fromConfig', () => { + it('should allow components by default', () => { + const enforcer = CatalogRulesEnforcer.fromConfig(new ConfigReader({})); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(true); + }); + + it('should deny all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ catalog: { rules: [] } }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + + it('should allow all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['User', 'Group'] }, { allow: ['Component'] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { rules: [{ allow: ['User'] }, { allow: ['Component'] }] }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + + it('should allow groups from a specific github location', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['user'] }], + locations: [ + { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + rules: [ + { + allow: ['Group'], + }, + ], + }, + ], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + + it('should not care about location configuration in catalog.rules', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['Group'], locations: [{ type: 'github' }] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts new file mode 100644 index 0000000000..e964524bc7 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -0,0 +1,177 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { LocationSpec, Entity } from '@backstage/catalog-model'; + +/** + * A structure for matching entities to a given rule. + */ +type EntityMatcher = { + kind: string; +}; + +/** + * A structure for matching locations to a given rule. + */ +type LocationMatcher = { + target?: string; + type: string; +}; + +/** + * Rules to apply to catalog entities + * + * An undefined list of matchers means match all, an empty list of matchers means match none + */ +type CatalogRule = { + allow: EntityMatcher[]; + locations?: LocationMatcher[]; +}; + +export class CatalogRulesEnforcer { + /** + * Default rules used by the catalog. + * + * Denies any location from specifying user or group entities. + */ + static readonly defaultRules: CatalogRule[] = [ + { + allow: ['Component', 'API', 'Location'].map(kind => ({ kind })), + }, + ]; + + /** + * Loads catalog rules from config. + * + * This reads `catalog.rules` and defaults to the default rules if no value is present. + * The value of the config should be a list of config objects, each with a single `allow` + * field which in turn is a list of entity kinds to allow. + * + * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog. + * + * It also reads in rules from `catalog.locations`, where each location can have a list + * of rules for that specific location, specified in a `rules` field. + * + * For example: + * + * ```yaml + * catalog: + * rules: + * - allow: [Component, API] + * + * locations: + * - type: github + * target: https://github.com/org/repo/blob/master/users.yaml + * rules: + * - allow: [User, Group] + * - type: github + * target: https://github.com/org/repo/blob/master/systems.yaml + * rules: + * - allow: [System] + * ``` + */ + static fromConfig(config: Config) { + const rules = new Array(); + + if (config.has('catalog.rules')) { + const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ + allow: sub.getStringArray('allow').map(kind => ({ kind })), + })); + rules.push(...globalRules); + } else { + rules.push(...CatalogRulesEnforcer.defaultRules); + } + + if (config.has('catalog.locations')) { + const locationRules = config + .getConfigArray('catalog.locations') + .flatMap(locConf => { + if (!locConf.has('rules')) { + return []; + } + const type = locConf.getString('type'); + const target = locConf.getString('target'); + + return locConf.getConfigArray('rules').map(ruleConf => ({ + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: [{ type, target }], + })); + }); + + rules.push(...locationRules); + } + + return new CatalogRulesEnforcer(rules); + } + + constructor(private readonly rules: CatalogRule[]) {} + + /** + * Checks wether a specific entity/location combination is allowed + * according to the configured rules. + */ + isAllowed(entity: Entity, location: LocationSpec) { + for (const rule of this.rules) { + if (!this.matchLocation(location, rule.locations)) { + continue; + } + + if (this.matchEntity(entity, rule.allow)) { + return true; + } + } + + return false; + } + + private matchLocation( + location: LocationSpec, + matchers?: LocationMatcher[], + ): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (matcher.type !== location.type) { + continue; + } + if (matcher.target && matcher.target !== location.target) { + continue; + } + return true; + } + + return false; + } + + private matchEntity(entity: Entity, matchers?: EntityMatcher[]): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) { + continue; + } + + return true; + } + + return false; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 8ec575d6ec..8da6e20ab3 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Config, ConfigReader } from '@backstage/config'; import { Entity, EntityPolicies, @@ -26,9 +27,13 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; +import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; +import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; +import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; +import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; import * as result from './processors/results'; import { LocationProcessor, @@ -41,25 +46,44 @@ import { } from './processors/types'; import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; +import { CatalogRulesEnforcer } from './CatalogRules'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; +type Options = { + logger?: Logger; + config?: Config; + processors?: LocationProcessor[]; +}; + /** * Implements the reading of a location through a series of processor tasks. */ export class LocationReaders implements LocationReader { private readonly logger: Logger; private readonly processors: LocationProcessor[]; + private readonly rulesEnforcer: CatalogRulesEnforcer; - static defaultProcessors( - entityPolicy: EntityPolicy = new EntityPolicies(), - ): LocationProcessor[] { + static defaultProcessors(options: { + logger: Logger; + config?: Config; + entityPolicy?: EntityPolicy; + }): LocationProcessor[] { + const { + logger, + config = new ConfigReader({}, 'missing-config'), + entityPolicy = new EntityPolicies(), + } = options; return [ + StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GithubReaderProcessor(), - new GithubApiReaderProcessor(), + GithubReaderProcessor.fromConfig(config, logger), + new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), + new BitbucketApiReaderProcessor(config), + new AzureApiReaderProcessor(config), + new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), @@ -67,12 +91,16 @@ export class LocationReaders implements LocationReader { ]; } - constructor( - logger: Logger = getVoidLogger(), - processors: LocationProcessor[] = LocationReaders.defaultProcessors(), - ) { + constructor({ + logger = getVoidLogger(), + config, + processors = LocationReaders.defaultProcessors({ logger, config }), + }: Options) { this.logger = logger; this.processors = processors; + this.rulesEnforcer = config + ? CatalogRulesEnforcer.fromConfig(config) + : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules); } async read(location: LocationSpec): Promise { @@ -89,11 +117,20 @@ export class LocationReaders implements LocationReader { } else if (item.type === 'data') { await this.handleData(item, emit); } else if (item.type === 'entity') { - const entity = await this.handleEntity(item, emit); - output.entities.push({ - entity, - location: item.location, - }); + if (this.rulesEnforcer.isAllowed(item.entity, item.location)) { + const entity = await this.handleEntity(item, emit); + output.entities.push({ + entity, + location: item.location, + }); + } else { + output.errors.push({ + location: item.location, + error: new Error( + `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`, + ), + }); + } } else if (item.type === 'error') { await this.handleError(item, emit); output.errors.push({ diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 07e917b97b..4ebb2edc79 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -24,3 +24,4 @@ export type { ReadLocationError, ReadLocationResult, } from './types'; +export * from './processors'; diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts new file mode 100644 index 0000000000..9b234e3e75 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; + +describe('AzureApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + azureApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + + it('should build raw api', () => { + const processor = new AzureApiReaderProcessor(createConfig(undefined)); + const tests = [ + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + ), + err: undefined, + }, + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }, + }, + }, + { + token: '', + expect: { + headers: {}, + }, + err: + "Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string", + }, + { + token: undefined, + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + if (test.err) { + expect( + () => new AzureApiReaderProcessor(createConfig(test.token)), + ).toThrowError(test.err); + } else { + const processor = new AzureApiReaderProcessor(createConfig(test.token)); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..03ff30ea69 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.azureApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' || + !path.match(/\.yaml$/) + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'items', + ].join('/'); + + const queryParams = [`path=${path}`]; + + if (ref) { + queryParams.push(`version=${ref}`); + } + + url.search = queryParams.join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts new file mode 100644 index 0000000000..953f0703be --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; + +describe('BitbucketApiReaderProcessor', () => { + const createConfig = ( + username: string | undefined, + appPassword: string | undefined, + ) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + bitbucketApi: { + username: username, + appPassword: appPassword, + }, + }, + }, + }, + }, + ]); + + it('should build raw api', () => { + const processor = new BitbucketApiReaderProcessor( + createConfig(undefined, undefined), + ); + + const tests = [ + { + target: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + url: new URL( + 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + username: '', + password: '', + expect: { + headers: {}, + }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", + }, + { + username: 'only-user-provided', + password: '', + expect: { + headers: {}, + }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string", + }, + { + username: '', + password: 'only-password-provided', + expect: { + headers: {}, + }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", + }, + { + username: 'some-user', + password: 'my-secret', + expect: { + headers: { + Authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', + }, + }, + }, + { + username: undefined, + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: 'only-user-provided', + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: undefined, + password: 'only-password-provided', + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + if (test.err) { + expect( + () => + new BitbucketApiReaderProcessor( + createConfig(test.username, test.password), + ), + ).toThrowError(test.err); + } else { + const processor = new BitbucketApiReaderProcessor( + createConfig(test.username, test.password), + ); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts similarity index 69% rename from plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts rename to plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts index ff61d004ca..97ecf0cd1f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -18,17 +18,29 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; -export class GithubApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; +export class BitbucketApiReaderProcessor implements LocationProcessor { + private username: string; + private password: string; + + constructor(config: Config) { + this.username = + config.getOptionalString('catalog.processors.bitbucketApi.username') ?? + ''; + this.password = + config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? + ''; + } getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; + const headers: HeadersInit = {}; - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; + if (this.username !== '' && this.password !== '') { + headers.Authorization = `Basic ${Buffer.from( + `${this.username}:${this.password}`, + 'utf8', + ).toString('base64')}`; } const requestOptions: RequestInit = { @@ -43,7 +55,7 @@ export class GithubApiReaderProcessor implements LocationProcessor { optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'github/api') { + if (location.type !== 'bitbucket/api') { return false; } @@ -69,13 +81,13 @@ export class GithubApiReaderProcessor implements LocationProcessor { const message = `Unable to read ${location.type} ${location.target}, ${e}`; emit(result.generalError(location, message)); } - return true; } // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -84,34 +96,35 @@ export class GithubApiReaderProcessor implements LocationProcessor { empty, userOrOrg, repoName, - blobKeyword, + srcKeyword, ref, ...restOfPath ] = url.pathname.split('/'); if ( - url.hostname !== 'github.com' || + url.hostname !== 'bitbucket.org' || empty !== '' || userOrOrg === '' || repoName === '' || - blobKeyword !== 'blob' || + srcKeyword !== 'src' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL or Invalid file path'); + throw new Error('Wrong Bitbucket URL or Invalid file path'); } // transform to api url.pathname = [ empty, - 'repos', + '2.0', + 'repositories', userOrOrg, repoName, - 'contents', + 'src', + ref, ...restOfPath, ].join('/'); - url.hostname = 'api.github.com'; + url.hostname = 'api.bitbucket.org'; url.protocol = 'https'; - url.search = `ref=${ref}`; return url; } catch (e) { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts deleted file mode 100644 index 4dd3fd359f..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; - -describe('GithubApiReaderProcessor', () => { - it('should build raw api', () => { - const processor = new GithubApiReaderProcessor(); - - const tests = [ - { - target: 'https://github.com/a/b/blob/master/path/to/c.yaml', - url: new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - { - target: - 'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml', - url: new URL( - 'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master', - ), - err: undefined, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else { - expect(processor.buildRawUrl(test.target)).toEqual(test.url); - } - } - }); - - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - Accept: 'application/vnd.github.v3.raw', - Authorization: 'token 0123456789', - }, - }, - }, - { - token: '', - expect: { - headers: { - Accept: 'application/vnd.github.v3.raw', - }, - }, - }, - ]; - - for (const test of tests) { - process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubApiReaderProcessor(); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts new file mode 100644 index 0000000000..3ab24541e7 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -0,0 +1,269 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { + getApiRequestOptions, + getApiUrl, + getRawRequestOptions, + getRawUrl, + GithubReaderProcessor, + ProviderConfig, + readConfig, +} from './GithubReaderProcessor'; + +describe('GithubReaderProcessor', () => { + describe('getApiRequestOptions', () => { + it('sets the correct API version', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect((getApiRequestOptions(config).headers as any).Accept).toEqual( + 'application/vnd.github.v3.raw', + ); + }); + + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + }; + expect( + (getApiRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getApiRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getRawRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + }; + expect( + (getRawRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getRawRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getApiUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getApiUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + apiBaseUrl: 'https://ghe.mycompany.net/api/v3', + }; + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + }); + + describe('getRawUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }; + expect( + getRawUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + rawBaseUrl: 'https://ghe.mycompany.net/raw', + }; + expect( + getRawUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'), + ); + }); + }); + + describe('readConfig', () => { + function config( + providers: { target: string; apiBaseUrl?: string; token?: string }[], + ) { + return ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { processors: { github: { providers } } }, + }, + }, + ]); + } + + it('adds a default GitHub entry when missing', () => { + const output = readConfig(config([]), getVoidLogger()); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('injects the correct GitHub API base URL when missing', () => { + const output = readConfig( + config([{ target: 'https://github.com' }]), + getVoidLogger(), + ); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('rejects custom targets with no base URLs', () => { + expect(() => + readConfig( + config([{ target: 'https://ghe.company.com' }]), + getVoidLogger(), + ), + ).toThrow( + 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', + ); + }); + + it('rejects funky configs', () => { + expect(() => + readConfig(config([{ target: 7 } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig( + config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/apiBaseUrl/); + expect(() => + readConfig( + config([{ target: 'https://github.com', token: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/token/); + }); + }); + + describe('implementation', () => { + it('rejects unknown types', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'not-github/api', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'github/api', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index b83c9a16f3..4f0c66148f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,26 +15,238 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; + } + async readLocation( location: LocationSpec, optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'github') { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { return false; } - try { - const url = this.buildRawUrl(location.target); + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } - // TODO(freben): Should "hard" errors thrown by this line be treated as - // notFound instead of fatal? - const response = await fetch(url.toString()); + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + const response = await fetch(url.toString(), options); if (response.ok) { const data = await response.buffer(); @@ -56,41 +268,4 @@ export class GithubReaderProcessor implements LocationProcessor { return true; } - - // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } } diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts new file mode 100644 index 0000000000..48ed270049 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; + +describe('GitlabApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + gitlabApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + + it('should build raw api', () => { + const processor = new GitlabApiReaderProcessor(createConfig(undefined)); + + const tests = [ + { + target: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: new URL( + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: new URL( + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + url: new URL( + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/', + url: null, + err: + 'Incorrect url: https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/, Error: GitLab url does not end in .ya?ml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target, 12345)).toThrowError( + test.err, + ); + } else if (test.url) { + expect(processor.buildRawUrl(test.target, 12345).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + 'PRIVATE-TOKEN': '0123456789', + }, + }, + }, + { + token: '', + err: + "Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string", + expect: { + headers: { + 'PRIVATE-TOKEN': '', + }, + }, + }, + { + token: undefined, + expect: { + headers: { + 'PRIVATE-TOKEN': '', + }, + }, + }, + ]; + + for (const test of tests) { + if (test.err) { + expect( + () => new GitlabApiReaderProcessor(createConfig(test.token)), + ).toThrowError(test.err); + } else { + const processor = new GitlabApiReaderProcessor( + createConfig(test.token), + ); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts new file mode 100644 index 0000000000..067edba3d9 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +export class GitlabApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; + if (this.privateToken !== '') { + headers['PRIVATE-TOKEN'] = this.privateToken; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab/api') { + return false; + } + + try { + const projectID = await this.getProjectID(location.target); + const url = this.buildRawUrl(location.target, projectID); + const response = await fetch(url.toString(), this.getRequestOptions()); + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch + buildRawUrl(target: string, projectID: Number): URL { + try { + const url = new URL(target); + + const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + + if (!branchAndfilePath.match(/\.ya?ml$/)) { + throw new Error('GitLab url does not end in .ya?ml'); + } + + const [branch, ...filePath] = branchAndfilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + async getProjectID(target: string): Promise { + const url = new URL(target); + + if ( + // absPaths to gitlab files should contain /-/blob + // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + !url.pathname.match(/\/\-\/blob\//) + ) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Find ProjectID from url + // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' + // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + this.getRequestOptions(), + ); + const projectIDJson = await response.json(); + const projectID: Number = projectIDJson.id; + + return projectID; + } catch (e) { + throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts index 211f325afe..9f308ee184 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -77,7 +77,7 @@ export class GitlabReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong Gitlab URL'); + throw new Error('Wrong GitLab URL'); } // Replace 'blob' with 'raw' diff --git a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts new file mode 100644 index 0000000000..6a2d1096cc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import * as result from './results'; +import { Config } from '@backstage/config'; +import { LocationProcessorEmit } from './types'; + +export class StaticLocationProcessor implements StaticLocationProcessor { + static fromConfig(config: Config): StaticLocationProcessor { + const locations: LocationSpec[] = []; + + const lConfigs = config.getOptionalConfigArray('catalog.locations') ?? []; + for (const lConfig of lConfigs) { + const type = lConfig.getString('type'); + const target = lConfig.getString('target'); + locations.push({ type, target }); + } + + return new StaticLocationProcessor(locations); + } + + constructor(private readonly staticLocations: LocationSpec[]) {} + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'bootstrap') { + return false; + } + + for (const staticLocation of this.staticLocations) { + emit(result.location(staticLocation, false)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts new file mode 100644 index 0000000000..e30b320bf4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReaderProcessor } from './UrlReaderProcessor'; +import { + LocationProcessorDataResult, + LocationProcessorResult, + LocationProcessorErrorResult, +} from './types'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; + +describe('UrlReaderProcessor', () => { + const mockApiOrigin = 'http://localhost:23000'; + const server = setupServer(); + + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + it('should load from url', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => + res(ctx.body('Hello')), + ), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorDataResult; + + expect(generated.type).toBe('data'); + expect(generated.location).toBe(spec); + expect(generated.data.toString('utf8')).toBe('Hello'); + }); + + it('should fail load from url with error', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component-notfound.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component-notfound.yaml`, (_, res, ctx) => { + return res(ctx.status(404)); + }), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorErrorResult; + + expect(generated.type).toBe('error'); + expect(generated.location).toBe(spec); + expect(generated.error.name).toBe('NotFoundError'); + expect(generated.error.message).toBe( + `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`, + ); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts new file mode 100644 index 0000000000..0c879ea70c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class UrlReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'url') { + return false; + } + + try { + const response = await fetch(location.target); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts new file mode 100644 index 0000000000..9f1ede3f9e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { YamlProcessor } from './YamlProcessor'; +import { Entity } from '@backstage/catalog-model'; +import yaml from 'yaml'; +import { TextEncoder } from 'util'; +import { + LocationProcessorEntityResult, + LocationProcessorErrorResult, +} from './types'; + +describe('YamlProcessor', () => { + const processor = new YamlProcessor(); + const locationSpec = { + type: 'url', + target: 'http://example.com/component.yaml', + }; + + function encodeEntity(entity: string): Buffer { + const data = new TextEncoder().encode(entity); + return Buffer.from(data); + } + + it('should only process files with yaml', async () => { + const wrongLocationSpec = { + type: 'url', + target: 'http://example.com/component.json', + }; + + const buffer = Buffer.from([]); + const never = jest.fn(); + + expect(await processor.parseData(buffer, wrongLocationSpec, never)).toBe( + false, + ); + + expect(never).not.toBeCalled(); + }); + + it('should process url that contains yaml', async () => { + const containsYamlLocationSpec = { + type: 'url', + target: 'http://example.com/component?path=test.yaml&c=1&d=2', + }; + + const buffer = Buffer.from([]); + const emit = jest.fn(); + + expect( + await processor.parseData(buffer, containsYamlLocationSpec, emit), + ).toBe(true); + + expect(emit).toBeCalled(); + }); + + it('should process entity with yaml', async () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'my-component', + }, + spec: {}, + } as Entity; + + const buffer = encodeEntity(yaml.stringify(entity)); + const emit = jest.fn(); + + expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true); + + const e = emit.mock.calls[0][0] as LocationProcessorEntityResult; + expect(e.type).toBe('entity'); + expect(e.location).toBe(locationSpec); + expect(e.entity).toEqual(entity); + }); + + it('should process multiple entities with yaml', async () => { + const entityComponent = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'my-component', + }, + spec: {}, + } as Entity; + + const entityApi = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'my-api', + }, + spec: {}, + } as Entity; + + const buffer = encodeEntity( + `${yaml.stringify(entityComponent)}---\n${yaml.stringify(entityApi)}`, + ); + const emit = jest.fn(); + + expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true); + + const eComponent = emit.mock.calls[0][0] as LocationProcessorEntityResult; + expect(eComponent.type).toBe('entity'); + expect(eComponent.location).toBe(locationSpec); + expect(eComponent.entity).toEqual(entityComponent); + + const eApi = emit.mock.calls[1][0] as LocationProcessorEntityResult; + expect(eApi.type).toBe('entity'); + expect(eApi.location).toBe(locationSpec); + expect(eApi.entity).toEqual(entityApi); + }); + + it('should fail process entity on invalid yaml', async () => { + const buffer = encodeEntity('{'); + const emit = jest.fn(); + + expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true); + + const e = emit.mock.calls[0][0] as LocationProcessorErrorResult; + expect(e.error.message).toMatch(/^YAML error, /); + expect(e.type).toBe('error'); + expect(e.location).toBe(locationSpec); + }); + + it('should fail process entity if not object at root', async () => { + const buffer = encodeEntity('[]'); + const emit = jest.fn(); + + expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true); + + const e = emit.mock.calls[0][0] as LocationProcessorErrorResult; + expect(e.error.message).toMatch(/^Expected object at root, got /); + expect(e.type).toBe('error'); + expect(e.location).toBe(locationSpec); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 6a2b5cf419..79ae55fae1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor { location: LocationSpec, emit: LocationProcessorEmit, ): Promise { - if (!location.target.match(/\.ya?ml$/)) { + if (!location.target.match(/\.ya?ml/)) { return false; } diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts new file mode 100644 index 0000000000..ed5dc3bf8e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as results from './results'; + +export { results }; +export * from './types'; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 5c7ccb23f5..0b532d1664 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { createServiceBuilder } from '@backstage/backend-common'; +import { + createServiceBuilder, + loadBackendConfig, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { Server } from 'http'; import { Logger } from 'winston'; import { HigherOrderOperations } from '..'; @@ -34,12 +38,13 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); + const config = ConfigReader.fromConfigs(await loadBackendConfig()); logger.debug('Creating application...'); const db = await DatabaseManager.createInMemoryDatabase({ logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); - const locationReader = new LocationReaders(); + const locationReader = new LocationReaders({ logger, config }); const higherOrderOperation = new HigherOrderOperations( entitiesCatalog, locationsCatalog, diff --git a/plugins/catalog-backend/src/setupTests.ts b/plugins/catalog-backend/src/setupTests.ts index f7b6ca962d..ba33cf996b 100644 --- a/plugins/catalog-backend/src/setupTests.ts +++ b/plugins/catalog-backend/src/setupTests.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -require('jest-fetch-mock').enableMocks(); - export {}; diff --git a/plugins/catalog-graphql/.eslintrc.js b/plugins/catalog-graphql/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/catalog-graphql/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/catalog-graphql/README.md b/plugins/catalog-graphql/README.md new file mode 100644 index 0000000000..911d4a401c --- /dev/null +++ b/plugins/catalog-graphql/README.md @@ -0,0 +1,11 @@ +# Catalog GraphQL Plugin + +## Getting Started + +This is the Catalog GraphQL plugin. + +It provides the `catalog` part of the GraphQL schema. + +To register it with the GraphQL backend, be sure to follow the [Getting Started](../graphql/README.md#getting-started) guide of the GraphQL plugin. + +