Merge remote-tracking branch 'upstream/master' into mcalus3/add-catalog-import-plugin

.changeset/breezy-cobras-deny.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+Clear sidebar search field once a search is executed

.changeset/clever-moons-shake.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added readTree support to AzureUrlReader

.changeset/cli-links.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+The CLI now detects and transforms linked packages. You can link in external packages by adding them to both the `lerna.json` and `package.json` workspace paths.

.changeset/cost-insights-tiny-llamas-perform.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-cost-insights': patch
+---
+
+truncate large percentages > 1000%

.changeset/curly-yaks-invite.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add support for reading groups and users from the Microsoft Graph API.

.changeset/dull-pans-sip.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-scaffolder': patch
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Move constructing the catalog-info.yaml URL for scaffolded components to the publishers

.changeset/empty-kids-look.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-register-component': patch
+---
+
+Remove catalog link on validate popup

.changeset/small-worms-check.md

@@ -0,0 +1,6 @@
+---
+'example-app': patch
+'@backstage/plugin-search': patch
+---
+
+Using the search field in the sidebar now navigates to the search result page.

.github/styles/vocab.txt

@@ -1,9 +1,9 @@
 abc
+andrewthauer
 Apdex
 api
 Api
 apis
-andrewthauer
 args
 asciidoc
 async
@@ -12,20 +12,22 @@ backrub
 Balachandran
 benjdlambert
 Bigtable
+Billett
 Blackbox
 bool
 boolean
+builtins
 Chai
 changeset
 changesets
 Changesets
-changset
 chanwit
 Chanwit
 cisphobia
 cissexist
 classname
 cli
+cloudbuild
 cncf
 codeblocks
 Codecov
@@ -68,6 +70,7 @@ github
 Github
 gitlab
 Gitlab
+Grafana
 graphql
 graphviz
 Gustavsson
@@ -79,6 +82,7 @@ horizontalpodautoscalers
 Hostname
 http
 https
+Iain
 img
 incentivised
 inlined
@@ -98,8 +102,8 @@ learnings
 lerna
 Lerna
 magiclink
-Maintainership
 mailto
+maintainership
 Malus
 md
 microsite
@@ -116,6 +120,7 @@ msw
 namespace
 namespaces
 Namespaces
+namespacing
 neuro
 newrelic
 nginx
@@ -171,9 +176,11 @@ scaffolded
 scaffolder
 Scaffolder
 semlas
+semver
 Serverless
 Sinon
 smartsymobls
+Snyk
 sparklines
 Spotifiers
 spotify
@@ -182,6 +189,7 @@ squidfunk
 src
 stefanalund
 subkey
+subtree
 superfences
 Superfences
 superset
@@ -199,11 +207,13 @@ theres
 toc
 tolerations
 Tolerations
+toolchain
 toolsets
 tooltip
 tooltips
 touchpoints
 ui
+untracked
 upvote
 url
 utils
@@ -212,15 +222,10 @@ Voi
 Wealthsimple
 Weaveworks
 Webpack
+www
+WWW
 xyz
 yaml
 Zalando
 Zhou
 Zolotusky
-Billett
-cloudbuild
-Grafana
-Iain
-Snyk
-www
-WWW

app-config.yaml

@@ -140,6 +140,19 @@ catalog:
       #       dn: ou=access,ou=groups,ou=example,dc=example,dc=net
       #       options:
       #         filter: (&(objectClass=some-group-class)(!(groupType=email)))
+    microsoftGraphOrg:
+      ### Example for how to add your Microsoft Graph tenant
+      #providers:
+      #  - target: https://graph.microsoft.com/v1.0
+      #    authority: https://login.microsoftonline.com
+      #    tenantId:
+      #      $env: MICROSOFT_GRAPH_TENANT_ID
+      #    clientId:
+      #      $env: MICROSOFT_GRAPH_CLIENT_ID
+      #    clientSecret:
+      #      $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN
+      #    userFilter: accountEnabled eq true and userType eq 'member'
+      #    groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
 
   locations:
     # Backstage example components

docs/FAQ.md

@@ -48,10 +48,9 @@ source candidates. (And we'll probably end up writing some brand new ones, too.)
 ### What's the roadmap for Backstage?
 
 We envision three phases, which you can learn about in
-[our project roadmap](https://github.com/backstage/backstage#project-roadmap).
-Even though the open source version of Backstage is relatively new compared to
-our internal version, we have already begun work on various aspects of all three
-phases. Looking at the
+[our project roadmap](overview/roadmap.md). Even though the open source version
+of Backstage is relatively new compared to our internal version, we have already
+begun work on various aspects of all three phases. Looking at the
 [milestones for active issues](https://github.com/backstage/backstage/milestones)
 will also give you a sense of our progress.
 
@@ -115,8 +114,7 @@ type of content. Plugins all use a common set of platform APIs and reusable UI
 components. Plugins can fetch data either from the backend or an API exposed
 through the proxy.
 
-Learn more about
-[the different components](https://github.com/backstage/backstage#overview) that
+Learn more about [the different components](overview/what-is-backstage.md) that
 make up Backstage.
 
 ### Do I have to write plugins in TypeScript?
@@ -126,17 +124,17 @@ APIs in TypeScript, but aren't forcing it on individual plugins.
 
 ### How do I find out if a plugin already exists?
 
-Before you write a plugin,
+You can browse and search for all available plugins in the
+[Plugin Marketplace](https://backstage.io/plugins).
+
+If you can't find it in the marketplace, before you write a plugin
 [search the plugin issues](https://github.com/backstage/backstage/issues?q=is%3Aissue+label%3Aplugin+)
-to see if it already exists or is in the works. If no one's thought of it yet,
-great! Open a new issue as
+to see if is in the works. If no one's thought of it yet, great! Open a new
+issue as
 [a plugin suggestion](https://github.com/backstage/backstage/issues/new/choose)
 and describe what your plugin will do. This will help coordinate our
 contributors' efforts and avoid duplicating existing functionality.
 
-You can browse and search for all available plugins in the
-[Plugin Marketplace](https://backstage.io/plugins).
-
 ### Which plugin is used the most at Spotify?
 
 By far, our most-used plugin is our TechDocs plugin, which we use for creating
@@ -182,6 +180,10 @@ comes to [deployment](https://backstage.io/docs/getting-started/deployment-k8s),
 the system integrator (typically, the infrastructure team in your organization)
 maintains Backstage in your own environment.
 
+For more information, see our
+[Owners](https://github.com/backstage/backstage/blob/master/OWNERS.md) and
+[Governance](https://github.com/backstage/backstage/blob/master/GOVERNANCE.md).
+
 ### Does Spotify provide a managed version of Backstage?
 
 No, this is not a service offering. We build the piece of software, and someone
@@ -215,14 +217,14 @@ data is shared with.
 Yes. The core frontend framework could be used for building any large-scale web
 application where (1) multiple teams are building separate parts of the app, and
 (2) you want the overall experience to be consistent. That being said, in
-[Phase 2](https://github.com/backstage/backstage#project-roadmap) of the project
-we will add features that are needed for developer portals and systems for
-managing software ecosystems. Our ambition will be to keep Backstage modular.
+[Phase 2](overview/roadmap.md) of the project we will add features that are
+needed for developer portals and systems for managing software ecosystems. Our
+ambition will be to keep Backstage modular.
 
 ### How can I get involved?
 
 Jump right in! Come help us fix some of the
-[early bugs and first issues](https://github.com/backstage/backstage/labels/good%20first%20issue)
+[early bugs and good first issues](https://github.com/backstage/backstage/contribute)
 or reach [a new milestone](https://github.com/backstage/backstage/milestones).
 Or write an open source plugin for Backstage, like this
 [Lighthouse plugin](https://github.com/backstage/backstage/tree/master/plugins/lighthouse).

docs/features/software-catalog/extending-the-model.md

@@ -1,7 +1,7 @@
 ---
 id: extending-the-model
 title: Extending the model
-description: Documentation on Extending the model
+description: Documentation on extending the catalog model
 ---
 
 The Backstage catalog [entity data model](descriptor-format.md) is based on the
@@ -28,63 +28,324 @@ Backstage comes with a number of catalog concepts out of the box:
 
 We'll list different possibilities for extending this below.
 
+## Adding a New apiVersion of an Existing Kind
+
+Example intents:
+
+> "I want to evolve this core kind, tweaking the semantics a bit so I will bump
+> the apiVersion a step"
+
+> "This core kind is a decent fit but we want to evolve it at will so we'll move
+> it to our own company's apiVersion space and use that instead of
+> `backstage.io`."
+
+The `backstage.io` apiVersion space is reserved for use by the Backstage
+maintainers. Please do not change or add versions within that space.
+
+If you add an [apiVersion](descriptor-format.md#apiversion-and-kind-required)
+space of your own, you are effectively branching out from the underlying kind
+and making your own. An entity kind is identified by the apiVersion + kind pair,
+so even though the resulting entity may be similar to the core one, there will
+be no guarantees that plugins will be able to parse or understand its data. See
+below about adding a new kind.
+
 ## Adding a New Kind
 
-> TODO: Fill in
+Example intents:
+
+> "The kinds that come with the package are lacking. I want to model this other
+> thing that is a poor fit for either of the builtins."
+
+> "This core kind is a decent fit but we want to evolve it at will so we'll move
+> it to our own company's apiVersion space and use that instead of
+> `backstage.io`."
+
+A [kind](descriptor-format.md#apiversion-and-kind-required) is an overarching
+family, or an idea if you will, of entities that also share a schema. Backstage
+comes with a number of builtin ones that we believe are useful for a large
+variety of needs that one may want to model in Backstage. The primary ambition
+is to map things to these kinds, but sometimes you may want or need to extend
+beyond them.
+
+Introducing a new apiVersion is basically the same as adding a new kind. Bear in
+mind that most plugins will be compiled against the builtin
+`@backstage/catalog-model` package and have expectations that kinds align with
+that.
+
+The catalog backend itself, from a storage and API standpoint, does not care
+about the kind of entities it stores. Extending with new kinds is mainly a
+matter of permitting them to pass validation when building the backend catalog
+using the `CatalogBuilder`, and then to make plugins be able to understand the
+new kind.
+
+For the consuming side, it's a different story. Adding a kind has a very large
+impact. The very foundation of Backstage is to attach behavior and views and
+functionality to entities that we ascribe some meaning to. There will be many
+places where code checks `if (kind === 'X')` for some hard coded `X`, and casts
+it to a concrete type that it imported from a package such as
+`@backstage/catalog-model`.
+
+If you want to model something that doesn't feel like a fit for either of the
+builtin kinds, feel free to reach out to the Backstage maintainers to discuss
+how to best proceed.
+
+If you end up adding that new kind, you must namespace its `apiVersion`
+accordingly with a prefix that makes sense, typically based on your organization
+name - e.g. `my-company.net/v1`. Also do pick a new `kind` identifier that does
+not collide with the builtin kinds.
 
 ## Adding a New Type of an Existing Kind
 
-Backstage natively supports tracking of the following component
-[`type`](descriptor-format.md)'s:
+Example intents:
 
-- Services
-- Websites
-- Libraries
-- Documentation
-- Other
+> "This is clearly a component, but it's of a type that doesn't quite fit with
+> the ones I've seen before."
 
-![](../../assets/software-catalog/bsc-extend.png)
+> "We don't call our teams "team", can't we put "flock" as the group type?"
 
-Since these types are likely not the only kind of software you will want to
-track in Backstage, it is possible to add your own software types that fit your
-organization's data model. Inside Spotify our model has grown significantly over
-the years, and now includes ML models, Apps, data pipelines and many more.
+Some entity kinds have a `type` field in its spec. This is where an organization
+are free to express the variety of entities within a kind. This field is
+expected to follow some taxonomy that makes sense for yourself. The chosen value
+may affect what operations and views are enabled in Backstage for that entity.
+Inside Spotify our model has grown significantly over the years, and our
+component types now include ML models, apps, data pipelines and many more.
 
 It might be tempting to put software that doesn't fit into any of the existing
-types into Other. There are a few reasons why we advise against this; firstly,
-we have found that it is preferred to match the conceptual model that your
-engineers have when describing your software. Secondly, Backstage helps your
-engineers manage their software by integrating the infrastructure tooling
-through plugins. Different plugins are used for managing different types of
-components.
+types into an Other catch-all type. There are a few reasons why we advise
+against this; firstly, we have found that it is preferred to match the
+conceptual model that your engineers have when describing your software.
+Secondly, Backstage helps your engineers manage their software by integrating
+the infrastructure tooling through plugins. Different plugins are used for
+managing different types of components.
 
 For example, the
 [Lighthouse plugin](https://github.com/backstage/backstage/tree/master/plugins/lighthouse)
 only makes sense for Websites. The more specific you can be in how you model
 your software, the easier it is to provide plugins that are contextual.
 
-> TODO: Fill in
+Adding a new type takes relatively little effort and carries little risk. Any
+type value is accepted by the catalog backend, but plugins may have to be
+updated if you want particular behaviors attached to that new type.
+
+## Changing the Validation Rules for The Entity Envelope or Metadata Fields
+
+Example intents:
+
+> "We want to import our old catalog but the default set of allowed characters
+> for a metadata.name are too strict."
+
+> "I want to change the rules for annotations so that I'm allowed to store any
+> data in annotation values, not just strings."
+
+After pieces of raw entity data have been read from a location, they are passed
+through a fixed number of so called `Validators`, as part of the entity policy
+check step. They ensure that the types and syntax of the base envelope and
+metadata make sense - in short, things that aren't entity-kind-specific. Some or
+all of these validators can be replaced when building the backend catalog using
+the `CatalogBuilder`.
+
+The risk and impact of this type of extension varies, based on what it is that
+you want to do. For example, extending the valid character set for kinds,
+namespaces and names can be fairly harmless, with a few notable exceptions -
+there is code that expects these to never ever contain a colon or slash, for
+example, and introducing URL-unsafe characters risks breaking plugins that
+aren't careful about encoding arguments. Supporting non-strings in annotations
+may be possible but has not yet been tried out in the real world - there is
+likely to be some level of plugin breakage that can be hard to predict.
+
+Before making this kind of extension, we recommend that you contact the
+Backstage maintainers or a support partner to discuss your use case.
 
 ## Changing the Validation Rules for Core Entity Fields
 
-> TODO: Fill in
+Example intent:
+
+> "I don't like that the owner is mandatory. I'd like it to be optional."
+
+After reading and policy-checked entity data from a location, it is sent through
+the processor chain looking for processors that implement the
+`validateEntityKind` step, to see that the data is of a known kind and abides by
+its schema. There is a builtin processor that implements this for all known core
+kinds and matches the data against their fixed validation schema. This processor
+can be replaced when building the backend catalog using the `CatalogBuilder`,
+with a processor of your own that validates the data differently.
+
+This type of extension is high risk, and may have high impact across the
+ecosystem depending on the type of change that is made. It is therefore not
+recommended in normal cases. There will be a large number of plugins and
+processors - and even the core itself - that make assumptions about the shape of
+the data and import the typescript data type from the `@backstage/catalog-model`
+package.
 
 ## Adding New Fields to the Metadata Object
 
-> TODO: Fill in
+Example intent:
+
+> "Our entities have this auxiliary property that I would like to express for
+> several entity kinds and it doesn't really fit as a spec field."
+
+The metadata object is currently left open for extension. Any unknown fields
+found in the metadata will just be stored verbatim in the catalog. However we
+want to caution against extending the metadata excessively. Firstly, you run the
+risk of colliding with future extensions to the model. Secondly, it is common
+that this type of extension lives more comfortably elsewhere - primarily in the
+metadata labels or annotations, but sometimes you even may want to make a new
+component type or similar instead.
+
+There are some situations where metadata can be the right place. If you feel
+that you have run into such a case and that it would apply to others, do feel
+free to contact the Backstage maintainers or a support partner to discuss your
+use case. Maybe we can extend the core model to benefit both you and others.
 
 ## Adding New Fields to the Spec Object of an Existing Kind
 
-> TODO: Fill in
+Example intent:
+
+> "The builtin Component kind is fine but we want to add an additional field to
+> the spec for describing whether it's in prod or staging."
+
+A kind's schema validation typically doesn't forbid "unknown" fields in an
+entity `spec`, and the catalog will happily store whatever is in it. So doing
+this will usually work from the catalog's point of view.
+
+Adding fields like this is subject to the same risks as mentioned about metadata
+extensions above. Firstly, you run the risk of colliding with future extensions
+to the model. Secondly, it is common that this type of extension lives more
+comfortably elsewhere - primarily in the metadata labels or annotations, but
+sometimes you even may want to make a new component type or similar instead.
+
+There are some situations where the spec can be the right place. If you feel
+that you have run into such a case and that it would apply to others, do feel
+free to contact the Backstage maintainers or a support partner to discuss your
+use case. Maybe we can extend the core model to benefit both you and others.
 
 ## Adding a New Annotation
 
-> TODO: Fill in
+Example intents:
+
+> "Our custom made build system has the concept of a named pipeline-set, and we
+> want to associate individual components with their corresponding pipeline-sets
+> so we can show their build status."
+
+> "We have an alerting system that automatically monitors service health, and
+> there's this integration key that binds the service to an alerts pool. We want
+> to be able to show the ongoing alerts for our services in Backstage so it'd be
+> nice to attach that integration key to the entity somehow."
+
+Annotations are mainly intended to be consumed by plugins, for feature detection
+or linking into external systems. Sometimes they are added by humans, but often
+they are automatically generated at ingestion time by processors. There is a set
+of [well-known annotations](well-known-annotations.md), but you are free to add
+additional ones. This carries no risk or impact to other systems as long as you
+abide by the following naming rules.
+
+- The `backstage.io` annotation prefix is reserved for use by the Backstage
+  maintainers. Reach out to us if you feel that you would like to make an
+  addition to that prefix.
+- Annotations that pertain to a well known third party system should ideally be
+  prefixed with a domain, in a way that makes sense to a reader and connects it
+  clearly to the system (or the maker of the system). For example, you might use
+  a `pagerduty.com` prefix for pagerduty related annotations, but maybe not
+  `ldap.com` for LDAP annotations since it's not directly affiliated with or
+  owned by an LDAP foundation/company/similar.
+- Annotations that have no prefix at all, are considered local to your Backstage
+  instance and can be used freely as such, but you should not make use of them
+  outside of your organization. For example, if you were to open source a plugin
+  that generates or consumes annotations, then those annotations must be
+  properly prefixed with your company domain or a domain that pertains to the
+  annotation at hand.
 
 ## Adding a New Label
 
-> TODO: Fill in
+Example intents:
+
+> "Our process reaping system wants to periodically scrape for components that
+> have a certain property."
+
+> "It'd be nice if our service owners could just tag their components somehow to
+> let the CD system know to automatically generate SRV records or not for that
+> service."
+
+Labels are mainly intended to be used for filtering of entities, by external
+systems that want to find entities that have some certain property. This is
+sometimes used for feature detection / selection. An example could be to add a
+label `deployments.my-company.net/register-srv: "true"`.
+
+At the time of writing this, the use of labels is very limited and we are still
+settling together with the community on how to best use them. If you feel that
+your use case fits the labels best, we would appreciate if you let the Backstage
+maintainers know.
+
+You are free to add labels. This carries no risk or impact to other systems as
+long as you abide by the following naming rules.
+
+- The `backstage.io` label prefix is reserved for use by the Backstage
+  maintainers. Reach out to us if you feel that you would like to make an
+  addition to that prefix.
+- Labels that pertain to a well known third party system should ideally be
+  prefixed with a domain, in a way that makes sense to a reader and connects it
+  clearly to the system (or the maker of the system). For example, you might use
+  a `pagerduty.com` prefix for pagerduty related labels, but maybe not
+  `ldap.com` for LDAP labels since it's not directly affiliated with or owned by
+  an LDAP foundation/company/similar.
+- Labels that have no prefix at all, are considered local to your Backstage
+  instance and can be used freely as such, but you should not make use of them
+  outside of your organization. For example, if you were to open source a plugin
+  that generates or consumes labels, then those labels must be properly prefixed
+  with your company domain or a domain that pertains to the label at hand.
 
 ## Adding a New Relation Type
 
-> TODO: Fill in
+Example intents:
+
+> "We have this concept of service maintainership, separate from ownership, that
+> we would like to make relations to individual users for."
+
+> We feel that we want to explicitly model the team-to-global-department mapping
+> as a relation, because it is core to our org setup and we frequently query for
+> it.
+
+Any processor can emit relations for entities as they are being processed, and
+new processors can be added when building the backend catalog using the
+`CatalogBuilder`. They can emit relations based on the entity data itself, or
+based on information gathered from elsewhere. Relations are directed and go from
+a source entity to a target entity. They are also tied to the entity that
+originated them - the one that was subject to processing when the relation was
+emitted. Relations may be dangling (referencing something that does not actually
+exist by that name in the catalog), and callers need to be aware of that.
+
+There is a set of [well-known relations](well-known-relations.md), but you are
+free to emit your own as well. You cannot change the fact that they are directed
+and have a source and target that have to be an
+[entity reference](references.md), but you can invent your own types. You do not
+have to make any changes to the catalog backend in order to accept new relation
+types.
+
+At the time of writing this, we do not have any namespacing/prefixing scheme for
+relation types. The type is also not validated to contain only some particular
+set of characters. Until rules for this are settled, you should stick to using
+only letters, dashes and digits, and to avoid collisions with future core
+relation types, you may want to prefix the type somehow. For example:
+`myCompany-maintainerOf` + `myCompany-maintainedBy`.
+
+If you have a suggestion for a relation type to be elevated to the core
+offering, reach out to the Backstage maintainers or a support partner.
+
+## Using a Well-Known Relation Type for a New Purpose
+
+Example intents:
+
+> "The ownerOf/ownedBy relation types sound like a good fit for expressing how
+> users are technical owners of our company specific ServiceAccount kind, and we
+> want to reuse those relation types for that."
+
+At the time of writing, this is uncharted territory. If the documented use of a
+relation states that one end of the relation commonly is a User or a Group, for
+example, then consumers are likely to have conditional statements on the form
+`if (x.kind === 'User') {} else {}`, which get confused when an unexpected kind
+appears.
+
+If you want to extend the use of an established relation type in a way that has
+an effect outside of your organization, reach out to the Backstage maintainers
+or a support partner to discuss risk/impact. It may even be that one end of the
+relation could be considered for addition to the core.

docs/features/software-catalog/well-known-annotations.md

@@ -190,9 +190,25 @@ metadata:
 ```
 
 The value of these annotations are the corresponding attributes that were found
-when ingestion the entity from LDAP. Not all of them may be present, depending
+when ingesting the entity from LDAP. Not all of them may be present, depending
 on what attributes that the server presented at ingestion time.
 
+### graph.microsoft.com/tenant-id, graph.microsoft.com/group-id, graph.microsoft.com/user-id
+
+```yaml
+# Example:
+metadata:
+  annotations:
+    graph.microsoft.com/tenant-id: 6902611b-ffc1-463f-8af3-4d5285dc057b
+    graph.microsoft.com/group-id: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21
+    graph.microsoft.com/user-id: 2de244b5-104b-4e8f-a3b8-dce3c31e54b6
+```
+
+The value of these annotations are the corresponding attributes that were found
+when ingesting the entity from the Microsoft Graph API. Not all of them may be
+present, depending on what attributes that the server presented at ingestion
+time.
+
 ### sonarqube.org/project-key
 
 ```yaml

docs/getting-started/create-an-app.md

@@ -38,6 +38,42 @@ app-folder is the name that was provided when prompted.
 Inside that directory, it will generate all the files and folder structure
 needed for you to run your app.
 
+### Linking in local Backstage packages
+
+It can often be useful to try out changes to the packages in the main Backstage
+repo within your own app. For example if you want to make modifications to
+`@backstage/core` and try them out in your app.
+
+To link in external packages, add them to your `package.json` and `lerna.json`
+workspace paths. These can be either relative or absolute paths with or without
+globs. For example:
+
+```json
+"packages": [
+  "packages/*",
+  "plugins/*",
+  "../backstage/packages/core", // New path added to work on @backstage/core
+],
+```
+
+Then reinstall packages to make yarn set up symlinks:
+
+```bash
+yarn install
+```
+
+With this in place you can now modify the `@backstage/core` package within the
+main repo, and have those changes be reflected and tested in your app. Simply
+run your app using `yarn start` as normal.
+
+Note that for backend packages you need to make sure that linked packages are
+not dependencies of any non-linked package. If you for example want to work on
+`@backstage/backend-common`, you need to also link in other backend plugins and
+packages that depend on `@backstage/backend-common`, or temporarily disable
+those plugins in your backend. This is because the transformation of backend
+module tree stops whenever a non-local package is encountered, and from that
+point node will `require` packages directly for that entire module subtree.
+
 ### Troubleshooting
 
 The create app command doesn't always work as expected, this is a collection of

docs/overview/stability-index.md

@@ -0,0 +1,384 @@
+---
+id: stability-index
+title: Stability Index
+description:
+  An overview of the commitment to stability for different parts of the
+  Backstage codebase.
+---
+
+## Overview
+
+The purpose of the Backstage Stability Index is to communicate the stability of
+various parts of the project. It is tracked using a scoring system where a
+higher score indicates a higher level of stability and is a commitment to
+smoother transitions between breaking changes. Importantly, the Stability Index
+does not supersede [semver](https://semver.org/), meaning we will still adhere
+to semver and only do breaking changes in minor releases as long as we are on
+`0.x`.
+
+Each package or section is assigned a stability score between 0 and 3, with each
+point building on top of the previous one:
+
+- **0** - Breaking changes are noted in the changelog, and documentation is
+  updated.
+- **1** - The changelog entry includes a clearly documented upgrade path,
+  providing guidance for how to migrate previous usage patterns to the new
+  version.
+- **2** - Breaking changes always include a deprecation phase where both the old
+  and the new APIs can be used in parallel. This deprecation must have been
+  released for at least two weeks before the deprecated API is removed in a
+  minor version bump.
+- **3** - The time limit for the deprecation is 3 months instead of two weeks.
+
+TL;DR:
+
+- **0** - There's a changelog entry.
+- **1** - There's a migration guide.
+- **2** - 2 weeks of deprecation.
+- **3** - 3 months of deprecation.
+
+## Packages
+
+### [`example-app`](https://github.com/backstage/backstage/tree/master/packages/app/)
+
+This is the `packages/app` package, and it serves as an example as well as
+utility for local development in the main Backstage repo.
+
+Stability: `N/A`
+
+### [`example-backend`](https://github.com/backstage/backstage/tree/master/packages/backend/)
+
+This is the `packages/backend` package, and it serves as an example as well as
+utility for local development in the main Backstage repo.
+
+Stability: `N/A`
+
+### [`backend-common`](https://github.com/backstage/backstage/tree/master/packages/backend-common/)
+
+A collection of common helpers to be used by both backend plugins, and for
+constructing backend packages.
+
+Stability: `1`
+
+### [`catalog-client`](https://github.com/backstage/backstage/tree/master/packages/catalog-client/)
+
+An HTTP client for interacting with the catalog backend. Usable both in frontend
+and Backend.
+
+Stability: `0`. This is a very new addition and we have some immediate changes
+planned.
+
+### [`catalog-model`](https://github.com/backstage/backstage/tree/master/packages/catalog-model/)
+
+Contains the core catalog model, and utilities for working with entities. Usable
+both in frontend and Backend.
+
+Stability: `2`. The catalog model is evolving, but because of the broad usage we
+
+want to ensure some stability.
+
+### [`cli`](https://github.com/backstage/backstage/tree/master/packages/cli/)
+
+The main toolchain used for Backstage development. The interface that is
+considered for stability are the various commands and options passed to those
+commands, as well as the environment variables read by the CLI. The build output
+may change over time and is not considered a breaking change unless it is likely
+to affect external tooling.
+
+Stability: `2`
+
+### [`cli-common`](https://github.com/backstage/backstage/tree/master/packages/cli-common/)
+
+Lightweight utilities used by the various Backstage CLIs, not intended for
+external use.
+
+Stability: `N/A`
+
+### [`config`](https://github.com/backstage/backstage/tree/master/packages/config/)
+
+Provides the logic and interfaces for reading static configuration.
+
+Stability: `2`
+
+### [`config-loader`](https://github.com/backstage/backstage/tree/master/packages/config-loader/)
+
+Used to load in static configuration, mainly for use by the CLI and
+@backstage/backend-common.
+
+Stability: `1`. Mainly intended for internal use.
+
+### [`core`](https://github.com/backstage/backstage/tree/master/packages/core/)
+
+#### Section: React Components
+
+All of the React components exported from `src/components/` and `src/layout/`
+
+Stability: `1`. These components have not received a proper review of the API,
+but we also want to ensure stability.
+
+#### Section: Plugin API
+
+The parts of the core API that are used by plugins, and the way plugins expose
+functionality to apps and other plugins. Includes for example `createPlugin`,
+`createRouteRef`, `createApiRef`.
+
+Stability: `2`. There are planned breaking changes around the way that plugins
+expose features and do routing. We still commit to keeping a short deprecation
+period so that plugins outside of the main repo have time to migrate.
+
+#### Section: App API
+
+The APIs used exclusively in the app, such as `createApp` and the system icons.
+
+Stability: `2`
+
+#### Section: Utility API Definitions
+
+The type declarations of the core Utility APIs.
+
+Stability: `2`. Changes to the Utility API type declarations need time to
+propagate.
+
+#### Section: Utility API Implementations
+
+The interfaces and default implementations for various Utility APIs, such as
+ErrorApi, IdentityApi, the auth APIs, etc.
+
+Stability: `1`. Most changes to the core utility APIs will not lead to
+widespread breaking changes since most apps rely on the default implementations.
+
+### [`core-api`](https://github.com/backstage/backstage/tree/master/packages/core-api/)
+
+The non-visual parts of @backstage/core. Everything in this packages is
+re-exported from @backstage/core, and this package should not be used directly.
+
+Stability: See @backstage/core
+
+### [`create-app`](https://github.com/backstage/backstage/tree/master/packages/create-app/)
+
+The CLI used to scaffold new Backstage projects.
+
+Stability: `2`
+
+### [`dev-utils`](https://github.com/backstage/backstage/tree/master/packages/dev-utils/)
+
+Provides utilities for developing plugins in isolation.
+
+Stability: `0`. This package is largely broken and needs updates.
+
+### [`docgen`](https://github.com/backstage/backstage/tree/master/packages/docgen/)
+
+Internal CLI utility for generating API Documentation.
+
+Stability: `N/A`
+
+### [`e2e-test`](https://github.com/backstage/backstage/tree/master/packages/e2e-test/)
+
+Internal CLI utility for running e2e tests.
+
+Stability: `N/A`
+
+### [`storybook`](https://github.com/backstage/backstage/tree/master/packages/storybook/)
+
+Internal storybook build for publishing stories to
+https://backstage.io/storybook
+
+Stability: `N/A`
+
+### [`test-utils`](https://github.com/backstage/backstage/tree/master/packages/test-utils/)
+
+Utilities for writing tests for Backstage plugins and apps.
+
+Stability: `2`
+
+### [`test-utils-core`](https://github.com/backstage/backstage/tree/master/packages/test-utils-core/)
+
+Internal testing utilities that are separated out for usage in
+@backstage/core-api. All exports are re-exported by @backstage/test-utils. This
+package should not be depended on directly.
+
+Stability: See @backstage/test-utils
+
+### [`theme`](https://github.com/backstage/backstage/tree/master/packages/theme/)
+
+The core Backstage MUI theme along with customization utilities.
+
+#### Section: TypeScript
+
+This is the TypeScript API exported by the theme package.
+
+Stability: `2`
+
+#### Section: Visual Theme
+
+The visual theme exported by the theme packages, where for example changing a
+color could be considered a breaking change.
+
+Stability: `1`
+
+## Plugins
+
+Plugins are rarely marked as stable as the `@backstage/core` plugin API is under
+heavy development.
+
+Many backend plugins are split into "REST API" and "TypeScript Interface"
+sections. The "TypeScript Interface" refers to the API used to integrate the
+plugin into the backend.
+
+Any plugin that is not listed below is untracked and can generally be considered
+unstable with a score of `0`. Open a Pull Request if you want your plugin to be
+added!
+
+### [`api-docs`](https://github.com/backstage/backstage/tree/master/plugins/api-docs/)
+
+Components to discover and display API entities as an extension to the catalog
+plugin.
+
+Stability: `0`
+
+### [`app-backend`](https://github.com/backstage/backstage/tree/master/plugins/app-backend/)
+
+A backend plugin that can be used to serve the frontend app and inject
+configuration.
+
+Stability: `2`
+
+### [`auth-backend`](https://github.com/backstage/backstage/tree/master/plugins/auth-backend/)
+
+A backend plugin that implements the backend portion of the various
+authentication flows used in Backstage.
+
+#### Section: REST API
+
+Stability: `2`
+
+#### Section: TypeScript Interface
+
+Stability: `1`
+
+### [`catalog`](https://github.com/backstage/backstage/tree/master/plugins/catalog/)
+
+The frontend plugin for the catalog, with the table and building blocks for the
+entity pages.
+
+Stability: `1`. We're planning some work to overhaul how entity pages are
+constructed.
+
+### [`catalog-backend`](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend/)
+
+The backend API for the catalog, also exposes the processing subsystem for
+customization of the catalog. Powers the @backstage/plugin-catalog frontend
+plugin.
+
+#### Section: REST API
+
+Stability: `1`. There are plans to remove and rework some endpoints.
+
+#### Section: TypeScript Interface
+
+Stability: `1`. There are plans to rework parts of the Processor interface.
+
+### [`catalog-graphql`](https://github.com/backstage/backstage/tree/master/plugins/catalog-graphql/)
+
+Provides the catalog schema and resolvers for the graphql backend.
+
+Stability: `0`. Under heavy development and subject to change.
+
+### [`explore`](https://github.com/backstage/backstage/tree/master/plugins/explore/)
+
+A frontend plugin that introduces the concept of exploring internal and external
+tooling in an organization.
+
+Stability: `0`. Only an example at the moment and not customizable.
+
+### [`graphiql`](https://github.com/backstage/backstage/tree/master/plugins/graphiql/)
+
+Integrates GraphiQL as a tool to browse GraphQL API endpoints inside Backstage.
+
+Stability: `1`
+
+### [`graphql`](https://github.com/backstage/backstage/tree/master/plugins/graphql/)
+
+A backend plugin that provides
+
+Stability: `0`. Under heavy development and subject to change.
+
+### [`kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/)
+
+The frontend component of the Kubernetes plugin, used to browse and visualize
+Kubernetes resources.
+
+Stability: `1`.
+
+### [`kubernetes-backend`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend/)
+
+The backend component of the Kubernetes plugin, used to fetch Kubernetes
+resources from clusters and associate them with entities in the Catalog.
+
+Stability: `1`.
+
+### [`proxy-backend`](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/)
+
+A backend plugin used to set up proxying to other endpoints based on static
+configuration.
+
+Stability: `1`
+
+### [`register-component`](https://github.com/backstage/backstage/tree/master/plugins/register-component/)
+
+A frontend plugin that allows the user to register entity locations in the
+catalog.
+
+Stability: `0`. This plugin is likely to be replaced by a generic entity import
+plugin instead.
+
+### [`scaffolder`](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/)
+
+The frontend scaffolder plugin where one can browse templates and initiate
+scaffolding jobs.
+
+Stability: `1`
+
+### [`scaffolder-backend`](https://github.com/backstage/backstage/tree/master/plugins/scaffolder-backend/)
+
+The backend scaffolder plugin that provides an implementation for templates in
+the catalog.
+
+Stability: `1`. There is planned work to rework the scaffolder in
+https://github.com/backstage/backstage/issues/2771.
+
+### [`tech-radar`](https://github.com/backstage/backstage/tree/master/plugins/tech-radar/)
+
+Visualize the your company's official guidelines of different areas of software
+development.
+
+Stability: `0`
+
+### [`techdocs`](https://github.com/backstage/backstage/tree/master/plugins/techdocs/)
+
+The frontend component of the TechDocs plugin, used to browse technical
+documentation of entities.
+
+Stability: `1`
+
+### [`techdocs-backend`](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/)
+
+The backend component of the TechDocs plugin, used to transform and serve
+TechDocs.
+
+Stability: `0`
+
+### [`user-settings`](https://github.com/backstage/backstage/tree/master/plugins/user-settings/)
+
+A frontend plugin that provides a page where the user can tweak various
+settings.
+
+Stability: `1`
+
+### [`welcome`](https://github.com/backstage/backstage/tree/master/plugins/welcome/)
+
+A plugin that can be used to welcome the user to Backstage.
+
+Stability: `0`. This used to be the start page for the example app, but has been
+replaced by the catalog plugin. It is still viewable at `/welcome` but may be
+removed.

microsite/data/plugins/aws-lambda.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io
 category: Monitoring
 description: View AWS Lambda functions for your components in Backstage.
 documentation: https://roadie.io/backstage/plugins/aws-lambda
-iconUrl: https://roadie.io/static/77f62f79e27ae8565496e4df7eef8be5/45f2b/logo.png
+iconUrl: https://roadie.io/images/logos/lambda.png
 npmPackageName: '@roadiehq/backstage-plugin-aws-lambda'

microsite/data/plugins/buildkite.yaml

@@ -0,0 +1,12 @@
+---
+title: Buildkite
+author: roadie.io
+authorUrl: https://roadie.io
+category: CI
+description: View Buildkite CI builds for your service in Backstage.
+documentation: https://roadie.io/backstage/plugins/buildkite
+iconUrl: https://roadie.io/images/logos/buildkite.png
+npmPackageName: '@roadiehq/backstage-plugin-buildkite'
+tags:
+  - ci
+  - cd

microsite/data/plugins/firebase-functions.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io/
 category: Monitoring
 description: View Firebase Functions details for your service in Backstage.
 documentation: https://roadie.io/backstage/plugins/firebase-functions
-iconUrl: https://roadie.io/static/49fb23200ad0eaa6703b4ddf75c78cf1/45f2b/logo-vertical.png
+iconUrl: https://roadie.io/images/logos/firebase.png
 npmPackageName: '@roadiehq/backstage-plugin-firebase-functions'

microsite/data/plugins/github-insights.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io
 category: Monitoring
 description: View GitHub Insights for your components in Backstage.
 documentation: https://roadie.io/backstage/plugins/github-insights
-iconUrl: https://roadie.io/static/2ad5123c425908efde0c922d707e737b/06c84/code-icon.png
+iconUrl: https://roadie.io/images/logos/insights.png
 npmPackageName: '@roadiehq/backstage-plugin-github-insights'

microsite/data/plugins/github-pull-requests.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io/
 category: CI
 description: View GitHub pull requests for your service in Backstage.
 documentation: https://roadie.io/backstage/plugins/github-pull-requests
-iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png
+iconUrl: https://roadie.io/images/logos/github.png
 npmPackageName: '@roadiehq/backstage-plugin-github-pull-requests'

microsite/data/plugins/security-insights.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io/
 category: Security
 description: View Security Insights for your components in Backstage.
 documentation: https://roadie.io/backstage/plugins/security-insights
-iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png
+iconUrl: https://roadie.io/images/logos/github.png
 npmPackageName: '@roadiehq/backstage-plugin-security-insights'

microsite/data/plugins/travis-ci.yaml

@@ -5,5 +5,5 @@ authorUrl: https://roadie.io/
 category: CI
 description: View Travis CI builds for your service in Backstage.
 documentation: https://roadie.io/backstage/plugins/travis-ci
-iconUrl: https://roadie.io/static/af2941eaf0af675facb281d566f42e14/45f2b/travis-ci-mascot-200x200.png
+iconUrl: https://roadie.io/images/logos/travis.png
 npmPackageName: '@roadiehq/backstage-plugin-travis-ci'

microsite/package.json

@@ -17,7 +17,7 @@
     "@spotify/prettier-config": "^9.0.0",
     "docusaurus": "^2.0.0-alpha.66",
     "js-yaml": "^3.14.0",
-    "prettier": "^2.0.5"
+    "prettier": "^2.2.0"
   },
   "prettier": "@spotify/prettier-config"
 }

microsite/sidebars.json

@@ -7,6 +7,7 @@
       "overview/vision",
       "overview/background",
       "overview/adopting",
+      "overview/stability-index",
       "overview/logos"
     ],
     "Getting Started": [

microsite/yarn.lock

@@ -5204,10 +5204,10 @@ prepend-http@^2.0.0:
   resolved "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
   integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=
 
-prettier@^2.0.5:
-  version "2.1.2"
-  resolved "https://registry.npmjs.org/prettier/-/prettier-2.1.2.tgz#3050700dae2e4c8b67c4c3f666cdb8af405e1ce5"
-  integrity sha512-16c7K+x4qVlJg9rEbXl7HEGmQyZlG4R9AgP+oHKRMsMsuk8s+ATStlf1NpDqyBI1HpVyfjLOeMhH2LvuNvV5Vg==
+prettier@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.0.tgz#8a03c7777883b29b37fb2c4348c66a78e980418b"
+  integrity sha512-yYerpkvseM4iKD/BXLYUkQV5aKt4tQPqaGW6EsZjzyu0r7sVZZNPJW4Y8MyKmicp6t42XUPcBVA+H6sB3gqndw==
 
 prismjs@^1.17.1:
   version "1.21.0"

packages/app/package.json

@@ -36,10 +36,10 @@
     "@material-ui/core": "^4.11.0",
     "@material-ui/icons": "^4.9.1",
     "@octokit/rest": "^18.0.0",
-    "@roadiehq/backstage-plugin-github-insights": "^0.2.14",
-    "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.2",
-    "@roadiehq/backstage-plugin-travis-ci": "^0.2.7",
-    "@roadiehq/backstage-plugin-buildkite": "^0.1.2",
+    "@roadiehq/backstage-plugin-github-insights": "^0.2.15",
+    "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.3",
+    "@roadiehq/backstage-plugin-travis-ci": "^0.2.8",
+    "@roadiehq/backstage-plugin-buildkite": "^0.1.3",
     "history": "^5.0.0",
     "prop-types": "^15.7.2",
     "react": "^16.12.0",

packages/app/src/components/Root/Root.tsx

@@ -33,12 +33,12 @@ import {
   SidebarContext,
   SidebarItem,
   SidebarDivider,
-  SidebarSearchField,
   SidebarSpace,
 } from '@backstage/core';
 import { NavLink } from 'react-router-dom';
 import { graphiQLRouteRef } from '@backstage/plugin-graphiql';
 import { Settings as SidebarSettings } from '@backstage/plugin-user-settings';
+import { SidebarSearch } from '@backstage/plugin-search';
 
 const useSidebarLogoStyles = makeStyles({
   root: {
@@ -73,17 +73,11 @@ const SidebarLogo: FC<{}> = () => {
   );
 };
 
-const handleSearch = (query: string): void => {
-  // XXX (@koroeskohr): for testing purposes
-  // eslint-disable-next-line no-console
-  console.log(query);
-};
-
 const Root: FC<{}> = ({ children }) => (
   <SidebarPage>
     <Sidebar>
       <SidebarLogo />
-      <SidebarSearchField onSearch={handleSearch} />
+      <SidebarSearch />
       <SidebarDivider />
       {/* Global nav, not org-specific */}
       <SidebarItem icon={HomeIcon} to="/catalog" text="Home" />

packages/app/src/components/catalog/EntityPage.test.tsx

@@ -18,7 +18,7 @@ import { CICDSwitcher } from './EntityPage';
 import { UrlPatternDiscovery, ApiProvider, ApiRegistry } from '@backstage/core';
 import {
   buildKiteApiRef,
-  BuildKiteApi,
+  BuildkiteApi,
 } from '@roadiehq/backstage-plugin-buildkite';
 import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils';
 
@@ -42,11 +42,11 @@ describe('EntityPage Test', () => {
   const discoveryApi = UrlPatternDiscovery.compile('http://exampleapi.com');
 
   const apis = ApiRegistry.from([
-    [buildKiteApiRef, new BuildKiteApi({ discoveryApi })],
+    [buildKiteApiRef, new BuildkiteApi({ discoveryApi })],
   ]);
 
   describe('CICDSwitcher Test', () => {
-    it('Should render BuildKite View', async () => {
+    it('Should render Buildkite View', async () => {
       const renderedComponent = await renderWithEffects(
         wrapInTestApp(
           <ApiProvider apis={apis}>

packages/app/src/components/catalog/EntityPage.tsx

@@ -67,8 +67,8 @@ import {
   PullRequestsStatsCard,
 } from '@roadiehq/backstage-plugin-github-pull-requests';
 import {
-  Router as BuildKiteRouter,
-  isPluginApplicableToEntity as isBuildKiteAvailable,
+  Router as BuildkiteRouter,
+  isPluginApplicableToEntity as isBuildkiteAvailable,
 } from '@roadiehq/backstage-plugin-buildkite';
 
 export const CICDSwitcher = ({ entity }: { entity: Entity }) => {
@@ -77,8 +77,8 @@ export const CICDSwitcher = ({ entity }: { entity: Entity }) => {
   switch (true) {
     case isJenkinsAvailable(entity):
       return <JenkinsRouter entity={entity} />;
-    case isBuildKiteAvailable(entity):
-      return <BuildKiteRouter entity={entity} />;
+    case isBuildkiteAvailable(entity):
+      return <BuildkiteRouter entity={entity} />;
     case isGitHubActionsAvailable(entity):
       return <GitHubActionsRouter entity={entity} />;
     case isCircleCIAvailable(entity):

packages/app/src/plugins.ts

@@ -39,5 +39,5 @@ export { plugin as CostInsights } from '@backstage/plugin-cost-insights';
 export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights';
 export { plugin as CatalogImport } from '@backstage/plugin-catalog-import';
 export { plugin as UserSettings } from '@backstage/plugin-user-settings';
-export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite';
+export { plugin as Buildkite } from '@roadiehq/backstage-plugin-buildkite';
 export { plugin as Search } from '@backstage/plugin-search';

packages/backend-common/package.json

@@ -36,6 +36,7 @@
     "@backstage/test-utils": "^0.1.3",
     "@types/cors": "^2.8.6",
     "@types/express": "^4.17.6",
+    "archiver": "^5.0.2",
     "compression": "^1.7.4",
     "concat-stream": "^2.0.0",
     "cors": "^2.8.5",
@@ -55,6 +56,7 @@
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
     "tar": "^6.0.5",
+    "unzipper": "^0.10.11",
     "winston": "^3.2.1"
   },
   "peerDependencies": {
@@ -67,6 +69,7 @@
   },
   "devDependencies": {
     "@backstage/cli": "^0.3.0",
+    "@types/archiver": "^3.1.1",
     "@types/compression": "^1.7.0",
     "@types/concat-stream": "^1.6.0",
     "@types/fs-extra": "^9.0.3",
@@ -78,6 +81,7 @@
     "@types/stoppable": "^1.1.0",
     "@types/supertest": "^2.0.8",
     "@types/tar": "^4.0.3",
+    "@types/unzipper": "^0.10.3",
     "@types/webpack-env": "^1.15.2",
     "@types/yaml": "^1.9.7",
     "get-port": "^5.1.1",

packages/backend-common/src/reading/AzureUrlReader.test.ts

@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
+import fs from 'fs';
+import path from 'path';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '../logging';
-import { AzureUrlReader } from './AzureUrlReader';
+import { AzureUrlReader, getDownloadUrl } from './AzureUrlReader';
 import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';
 
@@ -32,104 +34,165 @@ describe('AzureUrlReader', () => {
   const worker = setupServer();
   msw.setupDefaultHandlers(worker);
 
-  beforeEach(() => {
-    worker.use(
-      rest.get('*', (req, res, ctx) =>
-        res(
-          ctx.status(200),
-          ctx.json({
-            url: req.url.toString(),
-            headers: req.headers.getAllHeaders(),
-          }),
+  describe('read', () => {
+    beforeEach(() => {
+      worker.use(
+        rest.get('*', (req, res, ctx) =>
+          res(
+            ctx.status(200),
+            ctx.json({
+              url: req.url.toString(),
+              headers: req.headers.getAllHeaders(),
+            }),
+          ),
         ),
-      ),
-    );
-  });
-
-  const createConfig = (token?: string) =>
-    new ConfigReader(
-      {
-        integrations: { azure: [{ host: 'dev.azure.com', token }] },
-      },
-      'test-config',
-    );
-
-  it.each([
-    {
-      url:
-        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
-      }),
-    },
-    {
-      url:
-        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
-      }),
-    },
-    {
-      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
-      config: createConfig('0123456789'),
-      response: expect.objectContaining({
-        headers: expect.objectContaining({
-          authorization: 'Basic OjAxMjM0NTY3ODk=',
-        }),
-      }),
-    },
-    {
-      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
-      config: createConfig(undefined),
-      response: expect.objectContaining({
-        headers: expect.not.objectContaining({
-          authorization: expect.anything(),
-        }),
-      }),
-    },
-  ])('should handle happy path %#', async ({ url, config, response }) => {
-    const [{ reader }] = AzureUrlReader.factory({
-      config,
-      logger,
-      treeResponseFactory,
+      );
     });
 
-    const data = await reader.read(url);
-    const res = await JSON.parse(data.toString('utf-8'));
-    expect(res).toEqual(response);
-  });
+    const createConfig = (token?: string) =>
+      new ConfigReader(
+        {
+          integrations: { azure: [{ host: 'dev.azure.com', token }] },
+        },
+        'test-config',
+      );
 
-  it.each([
-    {
-      url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
-      config: createConfig(),
-      error:
-        'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
-    },
-    {
-      url: 'com/a/b/blob/master/path/to/c.yaml',
-      config: createConfig(),
-      error:
-        'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
-    },
-    {
-      url: '',
-      config: createConfig(''),
-      error:
-        "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string",
-    },
-  ])('should handle error path %#', async ({ url, config, error }) => {
-    await expect(async () => {
+    it.each([
+      {
+        url:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
+        }),
+      },
+      {
+        url:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
+        }),
+      },
+      {
+        url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+        config: createConfig('0123456789'),
+        response: expect.objectContaining({
+          headers: expect.objectContaining({
+            authorization: 'Basic OjAxMjM0NTY3ODk=',
+          }),
+        }),
+      },
+      {
+        url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+        config: createConfig(undefined),
+        response: expect.objectContaining({
+          headers: expect.not.objectContaining({
+            authorization: expect.anything(),
+          }),
+        }),
+      },
+    ])('should handle happy path %#', async ({ url, config, response }) => {
       const [{ reader }] = AzureUrlReader.factory({
         config,
         logger,
         treeResponseFactory,
       });
-      await reader.read(url);
-    }).rejects.toThrow(error);
+
+      const data = await reader.read(url);
+      const res = await JSON.parse(data.toString('utf-8'));
+      expect(res).toEqual(response);
+    });
+
+    it.each([
+      {
+        url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+        config: createConfig(),
+        error:
+          'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
+      },
+      {
+        url: 'com/a/b/blob/master/path/to/c.yaml',
+        config: createConfig(),
+        error:
+          'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+      },
+      {
+        url: '',
+        config: createConfig(''),
+        error:
+          "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string",
+      },
+    ])('should handle error path %#', async ({ url, config, error }) => {
+      await expect(async () => {
+        const [{ reader }] = AzureUrlReader.factory({
+          config,
+          logger,
+          treeResponseFactory,
+        });
+        await reader.read(url);
+      }).rejects.toThrow(error);
+    });
+  });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://dev.azure.com/organization/project/_apis/git/repositories/repository/items',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new AzureUrlReader(
+        {
+          host: 'dev.azure.com',
+        },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://dev.azure.com/organization/project/_git/repository',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(2);
+      const mkDocsFile = await files[1].content();
+      const indexMarkdownFile = await files[0].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+  });
+
+  describe('getDownloadUrl', () => {
+    it('do not add scopePath if no path is specified', async () => {
+      const result = getDownloadUrl(
+        'https://dev.azure.com/organization/project/_git/repository',
+      );
+
+      expect(result.searchParams.get('scopePath')).toBeNull();
+    });
+
+    it('add scopePath if a path is specified', async () => {
+      const result = getDownloadUrl(
+        'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs',
+      );
+      expect(result.searchParams.get('scopePath')).toEqual('docs');
+    });
   });
 });

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -19,22 +19,55 @@ import {
   readAzureIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
+import { Readable } from 'stream';
+import parseGitUri from 'git-url-parse';
 import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';
+import { ReadTreeResponseFactory } from './tree';
+
+export function getDownloadUrl(url: string): URL {
+  const {
+    name: repoName,
+    owner: project,
+    organization,
+    protocol,
+    resource,
+    filepath,
+  } = parseGitUri(url);
+
+  // scopePath will limit the downloaded content
+  // /docs will only download the docs folder and everything below it
+  // /docs/index.md will only download index.md but put it in the root of the archive
+  const scopePath = filepath
+    ? `&scopePath=${encodeURIComponent(filepath)}`
+    : '';
+
+  return new URL(
+    `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`,
+  );
+}
 
 export class AzureUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const configs = readAzureIntegrationConfigs(
       config.getOptionalConfigArray('integrations.azure') ?? [],
     );
     return configs.map(options => {
-      const reader = new AzureUrlReader(options);
+      const reader = new AzureUrlReader(options, { treeResponseFactory });
       const predicate = (url: URL) => url.host === options.host;
       return { reader, predicate };
     });
   };
 
-  constructor(private readonly options: AzureIntegrationConfig) {
+  constructor(
+    private readonly options: AzureIntegrationConfig,
+    private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
+  ) {
     if (options.host !== 'dev.azure.com') {
       throw Error(
         `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`,
@@ -64,8 +97,26 @@ export class AzureUrlReader implements UrlReader {
     throw new Error(message);
   }
 
-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('AzureUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const response = await fetch(
+      getDownloadUrl(url).toString(),
+      this.getRequestOptions({ Accept: 'application/zip' }),
+    );
+    if (!response.ok) {
+      const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    return this.deps.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      filter: options?.filter,
+    });
   }
 
   // Converts
@@ -127,8 +178,10 @@ export class AzureUrlReader implements UrlReader {
     }
   }
 
-  private getRequestOptions(): RequestInit {
-    const headers: HeadersInit = {};
+  private getRequestOptions(additionalHeaders?: {
+    [key: string]: string;
+  }): RequestInit {
+    const headers: HeadersInit = additionalHeaders ?? {};
 
     if (this.options.token) {
       headers.Authorization = `Basic ${Buffer.from(

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -207,7 +207,7 @@ export class GithubUrlReader implements UrlReader {
 
     const path = `${repoName}-${ref}/${filepath}`;
 
-    return this.deps.treeResponseFactory.fromArchive({
+    return this.deps.treeResponseFactory.fromTarArchive({
       // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want
       //               to stick to using that in exclusively backend code.
       stream: (response.body as unknown) as Readable,

packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts

@@ -18,7 +18,8 @@ import os from 'os';
 import { Readable } from 'stream';
 import { Config } from '@backstage/config';
 import { ReadTreeResponse } from '../types';
-import { ArchiveResponse } from './ArchiveResponse';
+import { TarArchiveResponse } from './TarArchiveResponse';
+import { ZipArchiveResponse } from './ZipArchiveResponse';
 
 type FromArchiveOptions = {
   // A binary stream of a tar archive.
@@ -39,8 +40,17 @@ export class ReadTreeResponseFactory {
 
   constructor(private readonly workDir: string) {}
 
-  async fromArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
-    return new ArchiveResponse(
+  async fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
+    return new TarArchiveResponse(
+      options.stream,
+      options.path ?? '',
+      this.workDir,
+      options.filter,
+    );
+  }
+
+  async fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
+    return new ZipArchiveResponse(
       options.stream,
       options.path ?? '',
       this.workDir,

packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts

@@ -17,13 +17,13 @@
 import fs from 'fs-extra';
 import mockFs from 'mock-fs';
 import { resolve as resolvePath } from 'path';
-import { ArchiveResponse } from './ArchiveResponse';
+import { TarArchiveResponse } from './TarArchiveResponse';
 
 const archiveData = fs.readFileSync(
   resolvePath(__filename, '../../__fixtures__/repo.tar.gz'),
 );
 
-describe('ArchiveResponse', () => {
+describe('TarArchiveResponse', () => {
   beforeEach(() => {
     mockFs({
       '/test-archive.tar.gz': archiveData,
@@ -38,7 +38,7 @@ describe('ArchiveResponse', () => {
   it('should read files', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
     const files = await res.files();
 
     expect(files).toEqual([
@@ -61,7 +61,7 @@ describe('ArchiveResponse', () => {
   it('should read files with filter', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+    const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
       path.endsWith('.yml'),
     );
     const files = await res.files();
@@ -79,14 +79,14 @@ describe('ArchiveResponse', () => {
   it('should read as archive and files', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp');
     const buffer = await res.archive();
 
     await expect(res.archive()).rejects.toThrow(
       'Response has already been read',
     );
 
-    const res2 = new ArchiveResponse(buffer, '', '/tmp');
+    const res2 = new TarArchiveResponse(buffer, '', '/tmp');
     const files = await res2.files();
 
     expect(files).toEqual([
@@ -109,7 +109,7 @@ describe('ArchiveResponse', () => {
   it('should extract entire archive into directory', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, '', '/tmp');
+    const res = new TarArchiveResponse(stream, '', '/tmp');
     const dir = await res.dir();
 
     await expect(
@@ -123,7 +123,7 @@ describe('ArchiveResponse', () => {
   it('should extract archive into directory with a subpath', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
+    const res = new TarArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
     const dir = await res.dir();
 
     expect(dir).toMatch(/^\/tmp\/.*$/);
@@ -135,7 +135,7 @@ describe('ArchiveResponse', () => {
   it('should extract archive into directory with a subpath and filter', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+    const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
       path.endsWith('.yml'),
     );
     const dir = await res.dir({ targetDir: '/tmp' });

packages/backend-common/src/reading/tree/TarArchiveResponse.ts

@@ -34,7 +34,7 @@ const pipeline = promisify(pipelineCb);
 /**
  * Wraps a tar archive stream into a tree response reader.
  */
-export class ArchiveResponse implements ReadTreeResponse {
+export class TarArchiveResponse implements ReadTreeResponse {
   private read = false;
 
   constructor(
@@ -49,7 +49,7 @@ export class ArchiveResponse implements ReadTreeResponse {
       }
       if (subPath.startsWith('/')) {
         throw new TypeError(
-          `ArchiveResponse subPath must not start with a /, got '${subPath}'`,
+          `TarArchiveResponse subPath must not start with a /, got '${subPath}'`,
         );
       }
     }

packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import { resolve as resolvePath } from 'path';
+import { ZipArchiveResponse } from './ZipArchiveResponse';
+
+const archiveData = fs.readFileSync(
+  resolvePath(__filename, '../../__fixtures__/repo.zip'),
+);
+
+describe('ZipArchiveResponse', () => {
+  beforeEach(() => {
+    mockFs({
+      '/test-archive.zip': archiveData,
+      '/tmp': mockFs.directory(),
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should read files', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'docs/index.md',
+        content: expect.any(Function),
+      },
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      '# Test',
+      'site_name: Test',
+    ]);
+  });
+
+  it('should read files with filter', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const content = await files[0].content();
+    expect(content.toString('utf8').trim()).toEqual('site_name: Test');
+  });
+
+  it('should read as archive and files', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const buffer = await res.archive();
+
+    await expect(res.archive()).rejects.toThrow(
+      'Response has already been read',
+    );
+
+    const res2 = new ZipArchiveResponse(buffer, '', '/tmp');
+    const files = await res2.files();
+
+    expect(files).toEqual([
+      {
+        path: 'docs/index.md',
+        content: expect.any(Function),
+      },
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      '# Test',
+      'site_name: Test',
+    ]);
+  });
+
+  it('should extract entire archive into directory', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, '', '/tmp');
+    const dir = await res.dir();
+
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+
+  it('should extract archive into directory with a subpath', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
+    const dir = await res.dir();
+
+    expect(dir).toMatch(/^\/tmp\/.*$/);
+    await expect(
+      fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+
+  it('should extract archive into directory with a subpath and filter', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const dir = await res.dir({ targetDir: '/tmp' });
+
+    expect(dir).toBe('/tmp');
+    await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe(
+      true,
+    );
+    await expect(
+      fs.pathExists(resolvePath(dir, 'docs/index.md')),
+    ).resolves.toBe(false);
+  });
+});

packages/backend-common/src/reading/tree/ZipArchiveResponse.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import fs from 'fs-extra';
+import unzipper, { Entry } from 'unzipper';
+import archiver from 'archiver';
+import { Readable } from 'stream';
+import {
+  ReadTreeResponse,
+  ReadTreeResponseFile,
+  ReadTreeResponseDirOptions,
+} from '../types';
+
+/**
+ * Wraps a zip archive stream into a tree response reader.
+ */
+export class ZipArchiveResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: Readable,
+    private readonly subPath: string,
+    private readonly workDir: string,
+    private readonly filter?: (path: string) => boolean,
+  ) {
+    if (subPath) {
+      if (!subPath.endsWith('/')) {
+        this.subPath += '/';
+      }
+      if (subPath.startsWith('/')) {
+        throw new TypeError(
+          `ZipArchiveResponse subPath must not start with a /, got '${subPath}'`,
+        );
+      }
+    }
+  }
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  private getPath(entry: Entry): string {
+    return entry.path.slice(this.subPath.length);
+  }
+
+  private shouldBeIncluded(entry: Entry): boolean {
+    if (this.subPath) {
+      if (!entry.path.startsWith(this.subPath)) {
+        return false;
+      }
+    }
+    if (this.filter) {
+      return this.filter(this.getPath(entry));
+    }
+    return true;
+  }
+
+  async files(): Promise<ReadTreeResponseFile[]> {
+    this.onlyOnce();
+
+    const files = Array<ReadTreeResponseFile>();
+
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', (entry: Entry) => {
+        if (entry.type === 'Directory') {
+          entry.resume();
+          return;
+        }
+
+        if (this.shouldBeIncluded(entry)) {
+          files.push({
+            path: this.getPath(entry),
+            content: () => entry.buffer(),
+          });
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+
+    return files;
+  }
+
+  async archive(): Promise<Readable> {
+    this.onlyOnce();
+
+    if (!this.subPath) {
+      return this.stream;
+    }
+
+    const archive = archiver('zip');
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', (entry: Entry) => {
+        if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
+          archive.append(entry, { name: this.getPath(entry) });
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+    archive.finalize();
+
+    return archive;
+  }
+
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      options?.targetDir ??
+      (await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
+
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', async (entry: Entry) => {
+        // Ignore directory entries since we handle that with the file entries
+        // as a zip can have files with directories without directory entries
+        if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
+          const entryPath = this.getPath(entry);
+          const dirname = path.dirname(entryPath);
+          if (dirname) {
+            await fs.mkdirp(path.join(dir, dirname));
+          }
+          entry.pipe(fs.createWriteStream(path.join(dir, entryPath)));
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+
+    return dir;
+  }
+}

packages/cli/src/lib/bundler/config.ts

@@ -70,13 +70,27 @@ async function readBuildInfo() {
   };
 }
 
+async function loadLernaPackages(): Promise<
+  { name: string; location: string }[]
+> {
+  const LernaProject = require('@lerna/project');
+  const project = new LernaProject(cliPaths.targetDir);
+  return project.getPackages();
+}
+
 export async function createConfig(
   paths: BundlingPaths,
   options: BundlingOptions,
 ): Promise<webpack.Configuration> {
   const { checksEnabled, isDev, frontendConfig } = options;
 
-  const { plugins, loaders } = transforms(options);
+  const packages = await loadLernaPackages();
+  const { plugins, loaders } = transforms({
+    ...options,
+    externalTransforms: packages.map(({ name }) =>
+      cliPaths.resolveTargetRoot('node_modules', name),
+    ),
+  });
 
   const baseUrl = frontendConfig.getString('app.baseUrl');
   const validBaseUrl = new URL(baseUrl);
@@ -159,6 +173,10 @@ export async function createConfig(
       alias: {
         'react-dom': '@hot-loader/react-dom',
       },
+      // Enables proper resolution of packages when linking in external packages.
+      // Without this the packages would depend on dependencies in the node_modules
+      // of the external packages themselves, leading to module duplication
+      symlinks: false,
     },
     module: {
       rules: loaders,
@@ -181,17 +199,20 @@ export async function createBackendConfig(
 ): Promise<webpack.Configuration> {
   const { checksEnabled, isDev } = options;
 
-  const { loaders } = transforms(options);
-
   // Find all local monorepo packages and their node_modules, and mark them as external.
-  const LernaProject = require('@lerna/project');
-  const project = new LernaProject(cliPaths.targetDir);
-  const packages = await project.getPackages();
+  const packages = await await loadLernaPackages();
   const localPackageNames = packages.map((p: any) => p.name);
   const moduleDirs = packages.map((p: any) =>
     resolvePath(p.location, 'node_modules'),
   );
 
+  const { loaders } = transforms({
+    ...options,
+    externalTransforms: packages.map(({ name }) =>
+      cliPaths.resolveTargetRoot('node_modules', name),
+    ),
+  });
+
   return {
     mode: isDev ? 'development' : 'production',
     profile: false,
@@ -240,6 +261,7 @@ export async function createBackendConfig(
       alias: {
         'react-dom': '@hot-loader/react-dom',
       },
+      symlinks: false, // See frontend config, added here for the same reason
     },
     module: {
       rules: loaders,

packages/cli/src/lib/bundler/transforms.ts

@@ -16,7 +16,6 @@
 
 import webpack, { Module, Plugin } from 'webpack';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
-import { BundlingOptions, BackendBundlingOptions } from './types';
 import { svgrTemplate } from '../svgrTemplate';
 
 type Transforms = {
@@ -24,17 +23,25 @@ type Transforms = {
   plugins: Plugin[];
 };
 
-export const transforms = (
-  options: BundlingOptions | BackendBundlingOptions,
-): Transforms => {
-  const { isDev } = options;
+type TransformOptions = {
+  isDev: boolean;
+  // External paths that should be transformed
+  externalTransforms: string[];
+};
+
+export const transforms = (options: TransformOptions): Transforms => {
+  const { isDev, externalTransforms } = options;
 
   const extraTransforms = isDev ? ['react-hot-loader'] : [];
 
+  const transformExcludeCondition = {
+    and: [/node_modules/, { not: externalTransforms }],
+  };
+
   const loaders = [
     {
       test: /\.(tsx?)$/,
-      exclude: /node_modules/,
+      exclude: transformExcludeCondition,
       loader: require.resolve('@sucrase/webpack-loader'),
       options: {
         transforms: ['typescript', 'jsx', ...extraTransforms],
@@ -43,7 +50,7 @@ export const transforms = (
     },
     {
       test: /\.(jsx?|mjs)$/,
-      exclude: /node_modules/,
+      exclude: transformExcludeCondition,
       loader: require.resolve('@sucrase/webpack-loader'),
       options: {
         transforms: ['jsx', ...extraTransforms],

packages/core/package.json

@@ -60,7 +60,8 @@
     "react-sparklines": "^1.7.0",
     "react-syntax-highlighter": "^13.5.1",
     "react-use": "^15.3.3",
-    "remark-gfm": "^1.0.0"
+    "remark-gfm": "^1.0.0",
+    "zen-observable": "^0.8.15"
   },
   "devDependencies": {
     "@backstage/cli": "^0.3.0",

packages/core/src/components/AlertDisplay/AlertDisplay.test.tsx

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { AlertDisplay } from './AlertDisplay';
+import {
+  ApiProvider,
+  ApiRegistry,
+  alertApiRef,
+  AlertApiForwarder,
+} from '@backstage/core-api';
+import Observable from 'zen-observable';
+import { renderInTestApp } from '@backstage/test-utils';
+
+const TEST_MESSAGE = 'TEST_MESSAGE';
+
+describe('<AlertDisplay />', () => {
+  it('renders without exploding', async () => {
+    const apiRegistry = ApiRegistry.from([
+      [alertApiRef, new AlertApiForwarder()],
+    ]);
+
+    const { queryByText } = await renderInTestApp(
+      <ApiProvider apis={apiRegistry}>
+        <AlertDisplay />
+      </ApiProvider>,
+    );
+    expect(queryByText(TEST_MESSAGE)).not.toBeInTheDocument();
+  });
+
+  it('renders with message', async () => {
+    const apiRegistry = ApiRegistry.from([
+      [
+        alertApiRef,
+        {
+          post() {},
+          alert$() {
+            return Observable.of({ message: TEST_MESSAGE });
+          },
+        },
+      ],
+    ]);
+
+    const { queryByText } = await renderInTestApp(
+      <ApiProvider apis={apiRegistry}>
+        <AlertDisplay />
+      </ApiProvider>,
+    );
+
+    expect(queryByText(TEST_MESSAGE)).toBeInTheDocument();
+  });
+});

packages/core/src/components/EmptyState/EmptyState.test.tsx

@@ -34,6 +34,6 @@ describe('<EmptyState />', () => {
       rendered.getByText('Your plugin is missing an annotation'),
     ).toBeInTheDocument();
     expect(rendered.getByLabelText('button')).toBeInTheDocument();
-    expect(rendered.getByTestId('missingAnnotation')).toBeInTheDocument();
+    expect(rendered.getByAltText('annotation is missing')).toBeInTheDocument();
   });
 });

packages/core/src/components/EmptyState/EmptyStateImage.test.tsx

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils';
+import { EmptyStateImage } from './EmptyStateImage';
+
+describe('<EmptyStateImage />', () => {
+  it('render EmptyStateImage component with missing field', async () => {
+    const { getByAltText } = await renderWithEffects(
+      wrapInTestApp(<EmptyStateImage missing="field" />),
+    );
+    expect(getByAltText('annotation is missing')).toBeInTheDocument();
+  });
+
+  it('render EmptyStateImage component with missing info', async () => {
+    const { getByAltText } = await renderWithEffects(
+      wrapInTestApp(<EmptyStateImage missing="info" />),
+    );
+    expect(getByAltText('no Information')).toBeInTheDocument();
+  });
+
+  it('render EmptyStateImage component with missing content', async () => {
+    const { getByAltText } = await renderWithEffects(
+      wrapInTestApp(<EmptyStateImage missing="content" />),
+    );
+    expect(getByAltText('create Component')).toBeInTheDocument();
+  });
+
+  it('render EmptyStateImage component with missing data', async () => {
+    const { getByAltText } = await renderWithEffects(
+      wrapInTestApp(<EmptyStateImage missing="data" />),
+    );
+    expect(getByAltText('no Build')).toBeInTheDocument();
+  });
+});

packages/core/src/components/EmptyState/EmptyStateImage.tsx

@@ -45,7 +45,6 @@ export const EmptyStateImage = ({ missing }: Props) => {
           src={missingAnnotation}
           className={classes.generalImg}
           alt="annotation is missing"
-          data-testid="missingAnnotation"
         />
       );
     case 'info':

packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { act, fireEvent } from '@testing-library/react';
+import { renderInTestApp } from '@backstage/test-utils';
+import { FeatureCalloutCircular } from './FeatureCalloutCircular';
+
+const INITIAL_BOUNDING_RECT: DOMRect = {
+  width: 100,
+  height: 100,
+  x: 0,
+  y: 0,
+  bottom: 0,
+  left: 0,
+  right: 0,
+  top: 0,
+  toJSON: () => {},
+};
+
+const UPDATED_BOUNDING_RECT: DOMRect = {
+  width: 200,
+  height: 200,
+  x: 50,
+  y: 50,
+  bottom: 0,
+  left: 0,
+  right: 0,
+  top: 0,
+  toJSON: () => {},
+};
+
+beforeEach(() => {
+  Element.prototype.getBoundingClientRect = jest.fn(
+    () => INITIAL_BOUNDING_RECT,
+  );
+});
+
+describe('<FeatureCalloutCircular />', () => {
+  it('renders without exploding', async () => {
+    const rendered = await renderInTestApp(
+      <FeatureCalloutCircular
+        featureId="feature-id"
+        title="title"
+        description="description"
+      />,
+    );
+    rendered.getByText('description');
+    rendered.getByText('title');
+  });
+
+  it('renders with correct style', async () => {
+    const { getByTestId } = await renderInTestApp(
+      <FeatureCalloutCircular
+        featureId="feature-id"
+        title="title"
+        description="description"
+      />,
+    );
+    const dot = await getByTestId('dot');
+    const text = await getByTestId('text');
+
+    expect(dot).toBeInTheDocument();
+    expect(text).toBeInTheDocument();
+
+    // Dot style
+    expect(dot.style.left).toBe('-800px');
+    expect(dot.style.top).toBe('-800px');
+    expect(dot.style.width).toBe('1700px');
+    expect(dot.style.height).toBe('1700px');
+
+    // Text style
+    expect(text.style.left).toBe('-400px');
+    expect(text.style.top).toBe('120px');
+    expect(text.style.width).toBe('450px');
+  });
+
+  it('update when the user scrolls', async () => {
+    const { getByTestId } = await renderInTestApp(
+      <FeatureCalloutCircular
+        featureId="feature-id"
+        title="title"
+        description="description"
+      />,
+    );
+    const dot = await getByTestId('dot');
+    const text = await getByTestId('text');
+
+    act(() => {
+      Element.prototype.getBoundingClientRect = jest.fn(
+        () => UPDATED_BOUNDING_RECT,
+      );
+
+      // Trigger the window resize event.
+      fireEvent(window, new Event('resize'));
+    });
+
+    // Dot style
+    expect(dot.style.left).toBe('-750px');
+    expect(dot.style.top).toBe('-750px');
+    expect(dot.style.width).toBe('1800px');
+    expect(dot.style.height).toBe('1800px');
+
+    // Text style
+    expect(text.style.left).toBe('-300px');
+    expect(text.style.top).toBe('270px');
+    expect(text.style.width).toBe('450px');
+  });
+
+  it('update when the user resizes the window', async () => {
+    const { getByTestId } = await renderInTestApp(
+      <FeatureCalloutCircular
+        featureId="feature-id"
+        title="title"
+        description="description"
+      />,
+    );
+    const dot = await getByTestId('dot');
+    const text = await getByTestId('text');
+
+    act(() => {
+      Element.prototype.getBoundingClientRect = jest.fn(
+        () => UPDATED_BOUNDING_RECT,
+      );
+
+      // Trigger the window scroll event.
+      fireEvent(window, new Event('scroll'));
+    });
+
+    // Dot style
+    expect(dot.style.left).toBe('-750px');
+    expect(dot.style.top).toBe('-750px');
+    expect(dot.style.width).toBe('1800px');
+    expect(dot.style.height).toBe('1800px');
+
+    // Text style
+    expect(text.style.left).toBe('-300px');
+    expect(text.style.top).toBe('270px');
+    expect(text.style.width).toBe('450px');
+  });
+});

packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx

@@ -160,6 +160,7 @@ export const FeatureCalloutCircular: FC<Props> = ({
             <>
               <div
                 className={classes.dot}
+                data-testid="dot"
                 style={{
                   left: placement?.dotLeft,
                   top: placement?.dotTop,
@@ -176,6 +177,7 @@ export const FeatureCalloutCircular: FC<Props> = ({
               </div>
               <div
                 className={classes.text}
+                data-testid="text"
                 style={{
                   left: placement?.textLeft,
                   top: placement?.textTop,

packages/core/src/components/ProgressBars/LinearGauge.test.tsx

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { renderInTestApp } from '@backstage/test-utils';
+
+import { LinearGauge } from './LinearGauge';
+
+describe('<LinearGauge />', () => {
+  it('renders without exploding', async () => {
+    const { getByTitle } = await renderInTestApp(<LinearGauge value={0.5} />);
+    expect(getByTitle('50%')).toBeInTheDocument();
+  });
+
+  it('renders progress and title', async () => {
+    const { container } = await renderInTestApp(<LinearGauge value={0 / 0} />);
+    expect(container).toBeEmptyDOMElement();
+  });
+
+  it('renders with 100 as max value', async () => {
+    const { getByTitle } = await renderInTestApp(<LinearGauge value={1.5} />);
+    expect(getByTitle('100%')).toBeInTheDocument();
+  });
+});

packages/core/src/components/ProgressBars/LinearGauge.tsx

@@ -40,12 +40,14 @@ export const LinearGauge: FC<Props> = ({ value }) => {
   const strokeColor = getProgressColor(theme.palette, percent, false, 100);
   return (
     <Tooltip title={`${percent}%`}>
-      <Line
-        percent={percent}
-        strokeWidth={4}
-        trailWidth={4}
-        strokeColor={strokeColor}
-      />
+      <span>
+        <Line
+          percent={percent}
+          strokeWidth={4}
+          trailWidth={4}
+          strokeColor={strokeColor}
+        />
+      </span>
     </Tooltip>
   );
 };

packages/core/src/hooks/useQueryParamState.ts

@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 
+import { isEqual } from 'lodash';
 import qs from 'qs';
-import { useState } from 'react';
+import { useEffect, useState } from 'react';
 import { useLocation, useNavigate } from 'react-router-dom';
 import { useDebounce } from 'react-use';
 
@@ -64,6 +65,14 @@ export function useQueryParamState<T>(
     extractState(location.search, stateName),
   );
 
+  useEffect(() => {
+    const newState = extractState(location.search, stateName);
+
+    setQueryParamState(oldState =>
+      isEqual(newState, oldState) ? oldState : newState,
+    );
+  }, [location, stateName]);
+
   useDebounce(
     () => {
       const queryString = joinQueryString(

packages/core/src/layout/Sidebar/Items.tsx

@@ -221,6 +221,7 @@ export const SidebarSearchField: FC<SidebarSearchFieldProps> = props => {
   const handleEnter: KeyboardEventHandler = ev => {
     if (ev.key === 'Enter') {
       props.onSearch(input);
+      setInput('');
     }
   };
 
@@ -233,6 +234,7 @@ export const SidebarSearchField: FC<SidebarSearchFieldProps> = props => {
       <SidebarItem icon={SearchIcon}>
         <TextField
           placeholder="Search"
+          value={input}
           onChange={handleInput}
           onKeyDown={handleEnter}
           className={classes.searchContainer}

plugins/catalog-backend/package.json

@@ -20,6 +20,7 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@azure/msal-node": "^1.0.0-alpha.8",
     "@backstage/backend-common": "^0.3.0",
     "@backstage/catalog-model": "^0.2.0",
     "@backstage/config": "^0.1.1",
@@ -37,6 +38,7 @@
     "lodash": "^4.17.15",
     "morgan": "^1.10.0",
     "p-limit": "^3.0.2",
+    "qs": "^6.9.4",
     "sqlite3": "^5.0.0",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",

plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import {
+  MicrosoftGraphClient,
+  MicrosoftGraphProviderConfig,
+  readMicrosoftGraphConfig,
+  readMicrosoftGraphOrg,
+} from './microsoftGraph';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+/**
+ * Extracts teams and users out of an LDAP server.
+ */
+export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
+  private readonly providers: MicrosoftGraphProviderConfig[];
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
+    return new MicrosoftGraphOrgReaderProcessor({
+      ...options,
+      providers: c ? readMicrosoftGraphConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    providers: MicrosoftGraphProviderConfig[];
+    logger: Logger;
+  }) {
+    this.providers = options.providers;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'microsoft-graph-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(p.target),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
+      );
+    }
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info('Reading Microsoft Graph users and groups');
+
+    // We create a client each time as we need one that matches the specific provider
+    const client = MicrosoftGraphClient.create(provider);
+    const { users, groups } = await readMicrosoftGraphOrg(
+      client,
+      provider.tenantId,
+      {
+        userFilter: provider.userFilter,
+        groupFilter: provider.groupFilter,
+      },
+    );
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`,
+    );
+
+    // Done!
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -27,6 +27,7 @@ export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { OwnerRelationProcessor } from './OwnerRelationProcessor';
 export { LocationRefProcessor } from './LocationEntityProcessor';
+export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';
 export { PlaceholderProcessor } from './PlaceholderProcessor';
 export type { PlaceholderResolver } from './PlaceholderProcessor';
 export { StaticLocationProcessor } from './StaticLocationProcessor';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts

@@ -0,0 +1,323 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import { msw } from '@backstage/test-utils';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { MicrosoftGraphClient } from './client';
+
+describe('MicrosoftGraphClient', () => {
+  const confidentialClientApplication: jest.Mocked<msal.ConfidentialClientApplication> = {
+    acquireTokenByClientCredential: jest.fn(),
+  } as any;
+  let client: MicrosoftGraphClient;
+  const worker = setupServer();
+
+  msw.setupDefaultHandlers(worker);
+
+  beforeEach(() => {
+    confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue(
+      { token: 'ACCESS_TOKEN' } as any,
+    );
+    client = new MicrosoftGraphClient(
+      'https://example.com',
+      confidentialClientApplication,
+    );
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  it('should perform raw request', async () => {
+    worker.use(
+      rest.get('https://other.example.com/', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestRaw('https://other.example.com/');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledTimes(1);
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] });
+  });
+
+  it('should perform simple api request', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestApi('users');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+  });
+
+  it('should perform api request with filter, select and expand', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (req, res, ctx) =>
+        res(ctx.status(200), ctx.json({ queryString: req.url.search })),
+      ),
+    );
+
+    const response = await client.requestApi('users', {
+      filter: 'test eq true',
+      expand: ['children'],
+      select: ['id', 'children'],
+    });
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      queryString:
+        '?$filter=test%20eq%20true&$select=id,children&$expand=children',
+    });
+  });
+
+  it('should perform collection request for a single page', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first']);
+  });
+
+  it('should perform collection request for multiple pages', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+            '@odata.nextLink': 'https://example.com/users2',
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get('https://example.com/users2', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: ['second'] })),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first', 'second']);
+  });
+
+  it('should load user profile', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            surname: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const userProfile = await client.getUserProfile('user-id');
+
+    expect(userProfile).toEqual({ surname: 'Example' });
+  });
+
+  it('should throw expection if load user profile fails', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    await expect(() => client.getUserProfile('user-id')).rejects.toThrowError();
+  });
+
+  it('should load user profile photo with max size of 120', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              {
+                height: 120,
+                id: 120,
+              },
+              {
+                height: 500,
+                id: 500,
+              },
+            ],
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should not fail if user has no profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toBeFalsy();
+  });
+
+  it('should load profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) =>
+        res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load profile photo for size 120', async () => {
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id', '120');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load users', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ surname: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getUsers());
+
+    expect(values).toEqual([{ surname: 'Example' }]);
+  });
+
+  it('should load groups', async () => {
+    worker.use(
+      rest.get('https://example.com/groups', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ displayName: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getGroups());
+
+    expect(values).toEqual([{ displayName: 'Example' }]);
+  });
+
+  it('should load group members', async () => {
+    worker.use(
+      rest.get('https://example.com/groups/group-id/members', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              { '@odata.type': '#microsoft.graph.user' },
+              { '@odata.type': '#microsoft.graph.group' },
+            ],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.getGroupMembers('group-id'),
+    );
+
+    expect(values).toEqual([
+      { '@odata.type': '#microsoft.graph.user' },
+      { '@odata.type': '#microsoft.graph.group' },
+    ]);
+  });
+
+  it('should load organization', async () => {
+    worker.use(
+      rest.get('https://example.com/organization/tentant-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            displayName: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const organization = await client.getOrganization('tentant-id');
+
+    expect(organization).toEqual({ displayName: 'Example' });
+  });
+});
+
+async function collectAsyncIterable<T>(
+  iterable: AsyncIterable<T>,
+): Promise<T[]> {
+  const values = [];
+  for await (const value of iterable) {
+    values.push(value);
+  }
+  return values;
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts

@@ -0,0 +1,201 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
+import fetch from 'cross-fetch';
+import qs from 'qs';
+import { MicrosoftGraphProviderConfig } from './config';
+
+export type ODataQuery = {
+  filter?: string;
+  expand?: string[];
+  select?: string[];
+};
+
+export type GroupMember =
+  | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' })
+  | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' });
+
+export class MicrosoftGraphClient {
+  static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient {
+    const clientConfig: msal.Configuration = {
+      auth: {
+        clientId: config.clientId,
+        clientSecret: config.clientSecret,
+        authority: `${config.authority}/${config.tenantId}`,
+      },
+    };
+    const pca = new msal.ConfidentialClientApplication(clientConfig);
+    return new MicrosoftGraphClient(config.target, pca);
+  }
+
+  constructor(
+    private readonly baseUrl: string,
+    private readonly pca: msal.ConfidentialClientApplication,
+  ) {}
+
+  async *requestCollection<T>(
+    path: string,
+    query?: ODataQuery,
+  ): AsyncIterable<T> {
+    let response = await this.requestApi(path, query);
+
+    for (;;) {
+      if (response.status !== 200) {
+        await this.handleError(path, response);
+      }
+
+      const result = await response.json();
+      const elements: T[] = result.value;
+
+      yield* elements;
+
+      // Follow cursor to the next page if one is available
+      if (!result['@odata.nextLink']) {
+        return;
+      }
+
+      response = await this.requestRaw(result['@odata.nextLink']);
+    }
+  }
+
+  async requestApi(path: string, query?: ODataQuery): Promise<Response> {
+    const queryString = qs.stringify(
+      {
+        $filter: query?.filter,
+        $select: query?.select?.join(','),
+        $expand: query?.expand?.join(','),
+      },
+      {
+        addQueryPrefix: true,
+        // Microsoft Graph doesn't like an encoded query string
+        encode: false,
+      },
+    );
+
+    return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`);
+  }
+
+  async requestRaw(url: string): Promise<Response> {
+    // Make sure that we always have a valid access token (might be cached)
+    const token = await this.pca.acquireTokenByClientCredential({
+      scopes: ['https://graph.microsoft.com/.default'],
+    });
+
+    return await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${token.accessToken}`,
+      },
+    });
+  }
+
+  async getUserProfile(userId: string): Promise<MicrosoftGraph.User> {
+    const response = await this.requestApi(`users/${userId}`);
+
+    if (response.status !== 200) {
+      await this.handleError('user profile', response);
+    }
+
+    return await response.json();
+  }
+
+  async getUserPhotoWithSizeLimit(
+    userId: string,
+    maxSize: number,
+  ): Promise<string | undefined> {
+    const response = await this.requestApi(`users/${userId}/photos`);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError('user photos', response);
+    }
+
+    const result = await response.json();
+    const photos = result.value as MicrosoftGraph.ProfilePhoto[];
+    let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined;
+
+    // Find the biggest picture that is smaller than the max size
+    for (const p of photos) {
+      if (
+        !selectedPhoto ||
+        (p.height! >= selectedPhoto.height! && p.height! <= maxSize)
+      ) {
+        selectedPhoto = p;
+      }
+    }
+
+    if (!selectedPhoto) {
+      return undefined;
+    }
+
+    return await this.getUserPhoto(userId, selectedPhoto.id!);
+  }
+
+  async getUserPhoto(
+    userId: string,
+    sizeId?: string,
+  ): Promise<string | undefined> {
+    const path = sizeId
+      ? `users/${userId}/photos/${sizeId}/$value`
+      : `users/${userId}/photo/$value`;
+    const response = await this.requestApi(path);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError('photo', response);
+    }
+
+    return `data:image/jpeg;base64,${Buffer.from(
+      await response.arrayBuffer(),
+    ).toString('base64')}`;
+  }
+
+  async *getUsers(query?: ODataQuery): AsyncIterable<MicrosoftGraph.User> {
+    yield* this.requestCollection<MicrosoftGraph.User>(`users`, query);
+  }
+
+  async *getGroups(query?: ODataQuery): AsyncIterable<MicrosoftGraph.Group> {
+    yield* this.requestCollection<MicrosoftGraph.Group>(`groups`, query);
+  }
+
+  async *getGroupMembers(groupId: string): AsyncIterable<GroupMember> {
+    yield* this.requestCollection<GroupMember>(`groups/${groupId}/members`);
+  }
+
+  async getOrganization(
+    tenantId: string,
+  ): Promise<MicrosoftGraph.Organization> {
+    const response = await this.requestApi(`organization/${tenantId}`);
+
+    if (response.status !== 200) {
+      await this.handleError('organization/${tenantId}', response);
+    }
+
+    return await response.json();
+  }
+
+  private async handleError(path: string, response: Response): Promise<void> {
+    const result = await response.json();
+    const error = result.error as MicrosoftGraph.PublicError;
+
+    throw new Error(
+      `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`,
+    );
+  }
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readMicrosoftGraphConfig } from './config';
+
+describe('readMicrosoftGraphConfig', () => {
+  it('applies all of the defaults', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(
+      ConfigReader.fromConfigs([{ context: '', data: config }]),
+    );
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.microsoftonline.com',
+        userFilter: undefined,
+        groupFilter: undefined,
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+
+  it('reads all the values', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+          authority: 'https://login.example.com/',
+          userFilter: 'accountEnabled eq true',
+          groupFilter: 'securityEnabled eq false',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(
+      ConfigReader.fromConfigs([{ context: '', data: config }]),
+    );
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.example.com',
+        userFilter: 'accountEnabled eq true',
+        groupFilter: 'securityEnabled eq false',
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+});

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+export type MicrosoftGraphProviderConfig = {
+  target: string;
+  authority: string;
+  tenantId: string;
+  clientId: string;
+  clientSecret: string;
+  userFilter?: string;
+  groupFilter?: string;
+};
+
+export function readMicrosoftGraphConfig(
+  config: Config,
+): MicrosoftGraphProviderConfig[] {
+  const providers: MicrosoftGraphProviderConfig[] = [];
+  const providerConfigs = config.getOptionalConfigArray('providers') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    const authority =
+      providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') ||
+      'https://login.microsoftonline.com';
+    const tenantId = providerConfig.getString('tenantId');
+    const clientId = providerConfig.getString('clientId');
+    const clientSecret = providerConfig.getString('clientSecret');
+    const userFilter = providerConfig.getOptionalString('userFilter');
+    const groupFilter = providerConfig.getOptionalString('groupFilter');
+
+    providers.push({
+      target,
+      authority,
+      tenantId,
+      clientId,
+      clientSecret,
+      userFilter,
+      groupFilter,
+    });
+  }
+
+  return providers;
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The tenant id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION =
+  'graph.microsoft.com/tenant-id';
+
+/**
+ * The group id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION =
+  'graph.microsoft.com/group-id';
+
+/**
+ * The user id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { MicrosoftGraphClient } from './client';
+export type { MicrosoftGraphProviderConfig } from './config';
+export { readMicrosoftGraphConfig } from './config';
+export { readMicrosoftGraphOrg } from './read';
+export {
+  MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
+  MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+} from './constants';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts

@@ -0,0 +1,339 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import merge from 'lodash/merge';
+import { RecursivePartial } from '../../../util';
+import { GroupMember, MicrosoftGraphClient } from './client';
+import {
+  normalizeEntityName,
+  readMicrosoftGraphGroups,
+  readMicrosoftGraphOrganization,
+  readMicrosoftGraphUsers,
+  resolveRelations,
+} from './read';
+
+function user(data: RecursivePartial<UserEntity>): UserEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name: 'name' },
+      spec: { profile: {}, memberOf: [] },
+    } as UserEntity,
+    data,
+  );
+}
+
+function group(data: RecursivePartial<GroupEntity>): GroupEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: 'name',
+      },
+      spec: {
+        ancestors: [],
+        children: [],
+        descendants: [],
+        type: 'team',
+      },
+    } as GroupEntity,
+    data,
+  );
+}
+
+describe('read microsoft graph', () => {
+  const client: jest.Mocked<MicrosoftGraphClient> = {
+    getUsers: jest.fn(),
+    getGroups: jest.fn(),
+    getGroupMembers: jest.fn(),
+    getUserPhotoWithSizeLimit: jest.fn(),
+    getOrganization: jest.fn(),
+  } as any;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('normalizeEntityName', () => {
+    it('should normalize name to valid entity name', () => {
+      expect(normalizeEntityName('User Name')).toBe('user_name');
+    });
+
+    it('should normalize e-mail to valid entity name', () => {
+      expect(normalizeEntityName('user.name@example.com')).toBe(
+        'user.name_example.com',
+      );
+    });
+  });
+
+  describe('readMicrosoftGraphUsers', () => {
+    it('should read users', async () => {
+      async function* getExampleUsers() {
+        yield {
+          id: 'userid',
+          displayName: 'User Name',
+          mail: 'user.name@example.com',
+        };
+      }
+
+      client.getUsers.mockImplementation(getExampleUsers);
+      client.getUserPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { users } = await readMicrosoftGraphUsers(client, {
+        userFilter: 'accountEnabled eq true',
+      });
+
+      expect(users).toEqual([
+        user({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/user-id': 'userid',
+            },
+            name: 'user.name_example.com',
+          },
+          spec: {
+            profile: {
+              displayName: 'User Name',
+              email: 'user.name@example.com',
+              picture: 'data:image/jpeg;base64,...',
+            },
+          },
+        }),
+      ]);
+
+      expect(client.getUsers).toBeCalledTimes(1);
+      expect(client.getUsers).toBeCalledWith({
+        filter: 'accountEnabled eq true',
+        select: ['id', 'displayName', 'mail'],
+      });
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
+    });
+  });
+
+  describe('readMicrosoftGraphOrganization', () => {
+    it('should read organization', async () => {
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const { rootGroup } = await readMicrosoftGraphOrganization(
+        client,
+        'tenantid',
+      );
+
+      expect(rootGroup).toEqual(
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/tenant-id': 'tenantid',
+            },
+            name: 'organization_name',
+            description: 'Organization Name',
+          },
+          spec: {
+            type: 'root',
+          },
+        }),
+      );
+
+      expect(client.getOrganization).toBeCalledTimes(1);
+      expect(client.getOrganization).toBeCalledWith('tenantid');
+    });
+  });
+
+  describe('readMicrosoftGraphGroups', () => {
+    it('should read groups', async () => {
+      async function* getExampleGroups() {
+        yield {
+          id: 'groupid',
+          displayName: 'Group Name',
+        };
+      }
+
+      async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.group',
+          id: 'childgroupid',
+        };
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementation(getExampleGroupMembers);
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const {
+        groups,
+        groupMember,
+        groupMemberOf,
+        rootGroup,
+      } = await readMicrosoftGraphGroups(client, 'tenantid', {
+        groupFilter: 'securityEnabled eq false',
+      });
+
+      const expectedRootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenantid',
+          },
+          name: 'organization_name',
+          description: 'Organization Name',
+        },
+        spec: {
+          type: 'root',
+        },
+      });
+      expect(groups).toEqual([
+        expectedRootGroup,
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/group-id': 'groupid',
+            },
+            name: 'group_name',
+            description: 'Group Name',
+          },
+          spec: {
+            type: 'team',
+          },
+        }),
+      ]);
+      expect(rootGroup).toEqual(expectedRootGroup);
+      expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid']));
+      expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid']));
+      expect(groupMember.get('organization_name')).toEqual(new Set());
+
+      expect(client.getGroups).toBeCalledTimes(1);
+      expect(client.getGroups).toBeCalledWith({
+        filter: 'securityEnabled eq false',
+        select: ['id', 'displayName', 'mailNickname'],
+      });
+      expect(client.getGroupMembers).toBeCalledTimes(1);
+      expect(client.getGroupMembers).toBeCalledWith('groupid');
+    });
+  });
+
+  describe('resolveRelations', () => {
+    it('should resolve relations', async () => {
+      const rootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenant-id-root',
+          },
+          name: 'root',
+        },
+        spec: {
+          type: 'root',
+        },
+      });
+      const groupA = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-a',
+          },
+          name: 'a',
+        },
+      });
+      const groupB = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-b',
+          },
+          name: 'b',
+        },
+      });
+      const groupC = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-c',
+          },
+          name: 'c',
+        },
+      });
+      const user1 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-1',
+          },
+          name: 'user1',
+        },
+      });
+      const user2 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-2',
+          },
+          name: 'user2',
+        },
+      });
+      const groups = [rootGroup, groupA, groupB, groupC];
+      const users = [user1, user2];
+      const groupMember = new Map<string, Set<string>>();
+      groupMember.set('group-id-b', new Set(['group-id-c']));
+      const groupMemberOf = new Map<string, Set<string>>();
+      groupMemberOf.set('user-id-1', new Set(['group-id-a']));
+      groupMemberOf.set('user-id-2', new Set(['group-id-c']));
+
+      // We have a root groups
+      // We have three groups: a, b, c. c is child of b
+      // we have two users: u1, u2. u1 is member of a, u2 is member of c
+      resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+
+      expect(rootGroup.spec.parent).toBeUndefined();
+      expect(rootGroup.spec.ancestors).toEqual(expect.arrayContaining([]));
+      expect(rootGroup.spec.children).toEqual(
+        expect.arrayContaining(['a', 'b']),
+      );
+      expect(rootGroup.spec.descendants).toEqual(
+        expect.arrayContaining(['a', 'b', 'c']),
+      );
+
+      expect(groupA.spec.parent).toEqual('root');
+      expect(groupA.spec.ancestors).toEqual(expect.arrayContaining(['root']));
+      expect(groupA.spec.children).toEqual(expect.arrayContaining([]));
+      expect(groupA.spec.descendants).toEqual(expect.arrayContaining([]));
+
+      expect(groupB.spec.parent).toEqual('root');
+      expect(groupB.spec.ancestors).toEqual(expect.arrayContaining(['root']));
+      expect(groupB.spec.children).toEqual(expect.arrayContaining(['c']));
+      expect(groupB.spec.descendants).toEqual(expect.arrayContaining(['c']));
+
+      expect(groupC.spec.parent).toEqual('b');
+      expect(groupC.spec.ancestors).toEqual(
+        expect.arrayContaining(['root', 'b']),
+      );
+      expect(groupC.spec.children).toEqual(expect.arrayContaining([]));
+      expect(groupC.spec.descendants).toEqual(expect.arrayContaining([]));
+
+      expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a']));
+
+      expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c']));
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts

@@ -0,0 +1,342 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildMemberOf, buildOrgHierarchy } from '../util/org';
+import { MicrosoftGraphClient } from './client';
+import {
+  MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
+  MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+} from './constants';
+import limiterFactory from 'p-limit';
+
+export function normalizeEntityName(name: string): string {
+  return name
+    .trim()
+    .toLocaleLowerCase()
+    .replace(/[^a-zA-Z0-9_\-\.]/g, '_');
+}
+
+export async function readMicrosoftGraphUsers(
+  client: MicrosoftGraphClient,
+  options?: { userFilter?: string },
+): Promise<{
+  users: UserEntity[]; // With all relations empty
+}> {
+  const entities: UserEntity[] = [];
+  const picturePromises: Promise<void>[] = [];
+  const limiter = limiterFactory(10);
+
+  for await (const user of client.getUsers({
+    filter: options?.userFilter,
+    select: ['id', 'displayName', 'mail'],
+  })) {
+    if (!user.id || !user.displayName || !user.mail) {
+      continue;
+    }
+
+    const name = normalizeEntityName(user.mail);
+    const entity: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name,
+        annotations: {
+          [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!,
+        },
+      },
+      spec: {
+        profile: {
+          displayName: user.displayName!,
+          email: user.mail!,
+
+          // TODO: Additional fields?
+          // jobTitle: user.jobTitle || undefined,
+          // officeLocation: user.officeLocation || undefined,
+          // mobilePhone: user.mobilePhone || undefined,
+        },
+        memberOf: [],
+      },
+    };
+
+    // Download the photos in parallel, otherwise it can take quite some time
+    const loadPhoto = limiter(async () => {
+      entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit(
+        user.id!,
+        // We are limiting the photo size, as users with full resolution photos
+        // can make the Backstage API slow
+        120,
+      );
+    });
+
+    picturePromises.push(loadPhoto);
+    entities.push(entity);
+  }
+
+  // Wait for all photos to be downloaded
+  await Promise.all(picturePromises);
+
+  return { users: entities };
+}
+
+export async function readMicrosoftGraphOrganization(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+): Promise<{
+  rootGroup: GroupEntity; // With all relations empty
+}> {
+  // For now we expect a single root orgranization
+  const organization = await client.getOrganization(tenantId);
+  const name = normalizeEntityName(organization.displayName!);
+  const rootGroup: GroupEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: {
+      name: name,
+      description: organization.displayName!,
+      annotations: {
+        [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!,
+      },
+    },
+    spec: {
+      type: 'root',
+      ancestors: [],
+      children: [],
+      descendants: [],
+    },
+  };
+
+  return { rootGroup };
+}
+
+export async function readMicrosoftGraphGroups(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { groupFilter?: string },
+): Promise<{
+  groups: GroupEntity[]; // With all relations empty
+  rootGroup: GroupEntity | undefined; // With all relations empty
+  groupMember: Map<string, Set<string>>;
+  groupMemberOf: Map<string, Set<string>>;
+}> {
+  const groups: GroupEntity[] = [];
+  const groupMember: Map<string, Set<string>> = new Map();
+  const groupMemberOf: Map<string, Set<string>> = new Map();
+  const limiter = limiterFactory(10);
+
+  const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId);
+  groupMember.set(rootGroup.metadata.name, new Set<string>());
+  groups.push(rootGroup);
+
+  const groupMemberPromises: Promise<void>[] = [];
+
+  for await (const group of client.getGroups({
+    filter: options?.groupFilter,
+    select: ['id', 'displayName', 'mailNickname'],
+  })) {
+    if (!group.id || !group.displayName) {
+      continue;
+    }
+
+    const name = normalizeEntityName(group.mailNickname || group.displayName);
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: name,
+        description: group.displayName,
+        annotations: {
+          [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id,
+        },
+      },
+      spec: {
+        type: 'team',
+        // TODO: We could include a group email and picture
+        ancestors: [],
+        children: [],
+        descendants: [],
+      },
+    };
+
+    // Download the members in parallel, otherwise it can take quite some time
+    const loadGroupMembers = limiter(async () => {
+      for await (const member of client.getGroupMembers(group.id!)) {
+        if (!member.id) {
+          continue;
+        }
+
+        if (member['@odata.type'] === '#microsoft.graph.user') {
+          ensureItem(groupMemberOf, member.id, group.id!);
+        }
+
+        if (member['@odata.type'] === '#microsoft.graph.group') {
+          ensureItem(groupMember, group.id!, member.id);
+        }
+      }
+    });
+
+    groupMemberPromises.push(loadGroupMembers);
+    groups.push(entity);
+  }
+
+  // Wait for all group members to be loaded
+  await Promise.all(groupMemberPromises);
+
+  return {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  };
+}
+
+export function resolveRelations(
+  rootGroup: GroupEntity | undefined,
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMember: Map<string, Set<string>>,
+  groupMemberOf: Map<string, Set<string>>,
+) {
+  // Build reference lookup tables, we reference them by the id the the graph
+  const groupMap: Map<string, GroupEntity> = new Map(); // by group-id or tenant-id
+
+  for (const group of groups) {
+    if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION],
+        group,
+      );
+    }
+    if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION],
+        group,
+      );
+    }
+  }
+
+  // Resolve all member relationships into the reverse direction
+  const parentGroups = new Map<string, Set<string>>();
+
+  groupMember.forEach((members, groupId) =>
+    members.forEach(m => ensureItem(parentGroups, m, groupId)),
+  );
+
+  // Make sure every group (except root) has at least one parent. If the parent is missing, add the root.
+  if (rootGroup) {
+    const tenantId = rootGroup.metadata.annotations![
+      MICROSOFT_GRAPH_TENANT_ID_ANNOTATION
+    ];
+
+    groups.forEach(group => {
+      const groupId = group.metadata.annotations![
+        MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
+      ];
+
+      if (!groupId) {
+        return;
+      }
+
+      if (retrieveItems(parentGroups, groupId).size === 0) {
+        ensureItem(parentGroups, groupId, tenantId);
+        ensureItem(groupMember, tenantId, groupId);
+      }
+    });
+  }
+
+  groups.forEach(group => {
+    const id =
+      group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ??
+      group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION];
+
+    retrieveItems(groupMember, id).forEach(m => {
+      const childGroup = groupMap.get(m);
+      if (childGroup) {
+        group.spec.children.push(childGroup.metadata.name);
+      }
+    });
+
+    retrieveItems(parentGroups, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        // TODO: Only having a single parent group might not match every companies model, but fine for now.
+        group.spec.parent = parentGroup.metadata.name;
+      }
+    });
+  });
+
+  // Make sure that all groups have proper ancestors and descendants
+  buildOrgHierarchy(groups);
+
+  // Set relations for all users
+  users.forEach(user => {
+    const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION];
+
+    retrieveItems(groupMemberOf, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        user.spec.memberOf.push(parentGroup.metadata.name);
+      }
+    });
+  });
+
+  // Make sure all transitive memberships are available
+  buildMemberOf(groups, users);
+}
+
+export async function readMicrosoftGraphOrg(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { userFilter?: string; groupFilter?: string },
+): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
+  const { users } = await readMicrosoftGraphUsers(client, {
+    userFilter: options?.userFilter,
+  });
+  const {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  } = await readMicrosoftGraphGroups(client, tenantId, {
+    groupFilter: options?.groupFilter,
+  });
+
+  resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+  users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+  groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+
+  return { users, groups };
+}
+
+function ensureItem(
+  target: Map<string, Set<string>>,
+  key: string,
+  value: string,
+) {
+  let set = target.get(key);
+  if (!set) {
+    set = new Set();
+    target.set(key, set);
+  }
+  set!.add(value);
+}
+
+function retrieveItems(
+  target: Map<string, Set<string>>,
+  key: string,
+): Set<string> {
+  return target.get(key) ?? new Set();
+}

plugins/catalog-backend/src/ingestion/processors/util/org.test.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-import { GroupEntity } from '@backstage/catalog-model';
-import { buildOrgHierarchy } from './org';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildMemberOf, buildOrgHierarchy } from './org';
 
 function g(
   name: string,
@@ -67,3 +67,22 @@ describe('buildOrgHierarchy', () => {
     expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a']));
   });
 });
+
+describe('buildMemberOf', () => {
+  it('fills indirect member of groups', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const u: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name },
+      spec: { profile: {}, memberOf: ['c'] },
+    };
+
+    const groups = [a, b, c];
+    buildOrgHierarchy(groups);
+    buildMemberOf(groups, [u]);
+    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+  });
+});

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { GroupEntity } from '@backstage/catalog-model';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 
 export function buildOrgHierarchy(groups: GroupEntity[]) {
   const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
@@ -93,3 +93,22 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
     visitAncestors(group);
   }
 }
+
+// Ensure that users have their transitive group memberships. Requires that
+// the groups were previously processed with buildOrgHierarchy()
+export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+
+  users.forEach(user => {
+    const transitiveMemberOf = new Set([...user.spec.memberOf]);
+
+    user.spec.memberOf.forEach(groupName => {
+      const group = groupsByName.get(groupName);
+
+      if (group) {
+        group.spec.ancestors.forEach(g => transitiveMemberOf.add(g));
+      }
+    });
+    user.spec.memberOf = [...transitiveMemberOf];
+  });
+}

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -46,6 +46,7 @@ import {
   LocationReaders,
   LocationRefProcessor,
   OwnerRelationProcessor,
+  MicrosoftGraphOrgReaderProcessor,
   PlaceholderProcessor,
   PlaceholderResolver,
   StaticLocationProcessor,
@@ -283,6 +284,7 @@ export class CatalogBuilder {
         new FileReaderProcessor(),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
         LdapOrgReaderProcessor.fromConfig(config, { logger }),
+        MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),
         new UrlReaderProcessor({ reader, logger }),
         new CodeOwnersProcessor({ reader }),
         new LocationRefProcessor(),

plugins/catalog/package.json

@@ -46,6 +46,7 @@
     "@backstage/cli": "^0.3.0",
     "@backstage/dev-utils": "^0.1.4",
     "@backstage/test-utils": "^0.1.3",
+    "@microsoft/microsoft-graph-types": "^1.25.0",
     "@testing-library/jest-dom": "^5.10.1",
     "@testing-library/react": "^10.4.1",
     "@testing-library/react-hooks": "^3.3.0",

plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx

@@ -15,6 +15,7 @@
  */
 
 import React, { ReactNode, PropsWithChildren } from 'react';
+import classnames from 'classnames';
 import { Box, Divider, Typography } from '@material-ui/core';
 import { useTooltipStyles as useStyles } from '../../utils/styles';
 
@@ -35,6 +36,10 @@ export const BarChartTooltip = ({
   children,
 }: PropsWithChildren<BarChartTooltipProps>) => {
   const classes = useStyles();
+  const titleClassName = classnames(classes.truncate, {
+    [classes.maxWidth]: topRight === undefined,
+  });
+
   return (
     <Box className={classes.tooltip} display="flex" flexDirection="column">
       <Box
@@ -46,7 +51,7 @@ export const BarChartTooltip = ({
         pt={2}
       >
         <Box display="flex" flexDirection="column">
-          <Typography className={classes.truncate} variant="h6">
+          <Typography className={titleClassName} variant="h6">
             {title}
           </Typography>
           {subtitle && (
@@ -55,10 +60,10 @@ export const BarChartTooltip = ({
             </Typography>
           )}
         </Box>
-        {topRight && <Box ml={1}>{topRight}</Box>}
+        {topRight && <Box ml={2}>{topRight}</Box>}
       </Box>
       {content && (
-        <Box px={2} pt={2}>
+        <Box px={2} pt={2} className={classes.maxWidth}>
           <Typography variant="body1" paragraph>
             {content}
           </Typography>

plugins/cost-insights/src/utils/formatters.test.ts

@@ -16,6 +16,7 @@
 
 import {
   formatPeriod,
+  formatPercent,
   lengthyCurrencyFormatter,
   quarterOf,
 } from './formatters';
@@ -69,3 +70,17 @@ describe.each`
     expect(formatPeriod(duration, date, isEndDate)).toBe(output);
   });
 });
+
+describe.each`
+  ratio             | expected
+  ${0.0}            | ${'0%'}
+  ${0.000000000001} | ${'0%'}
+  ${-0.00000000001} | ${'0%'}
+  ${0.123123}       | ${'12%'}
+  ${1.123}          | ${'112%'}
+  ${10.123}         | ${'>1000%'}
+`('formatPercent', ({ ratio, expected }) => {
+  it(`correctly formats ${ratio} as ${expected}`, () => {
+    expect(formatPercent(ratio)).toBe(expected);
+  });
+});

plugins/cost-insights/src/utils/formatters.ts

@@ -83,9 +83,11 @@ export function formatPercent(n: number): string {
   if (isNaN(n) || Math.abs(n) < 0.01) {
     return '0%';
   }
-  if (Math.abs(n) >= 1e19) {
-    return '∞%';
+
+  if (Math.abs(n) > 10) {
+    return `>1000%`;
   }
+
   return `${(n * 100).toFixed(0)}%`;
 }
 

plugins/cost-insights/src/utils/styles.ts

@@ -386,6 +386,9 @@ export const useTooltipStyles = makeStyles<CostInsightsTheme>(
         boxShadow: theme.shadows[1],
         color: theme.palette.tooltip.color,
         fontSize: theme.typography.fontSize,
+        minWidth: 300,
+      },
+      maxWidth: {
         maxWidth: 300,
       },
       actions: {

plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.tsx

@@ -100,7 +100,9 @@ export const RegisterComponentResultDialog = ({
                     metadata={{
                       name: entity.metadata.name,
                       type: entity.spec.type,
-                      link: (
+                      link: dryRun ? (
+                        entityPath
+                      ) : (
                         <Link component={RouterLink} to={entityPath}>
                           {entityPath}
                         </Link>

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts

@@ -41,7 +41,7 @@ describe('Azure Publisher', () => {
   describe('publish: createRemoteInAzure', () => {
     it('should use azure-devops-node-api to create a repo in the given project', async () => {
       mockGitApi.createRepository.mockResolvedValue({
-        remoteUrl: 'mockclone',
+        remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
       } as { remoteUrl: string });
 
       const result = await publisher.publish({
@@ -52,7 +52,11 @@ describe('Azure Publisher', () => {
         directory: '/tmp/test',
       });
 
-      expect(result).toEqual({ remoteUrl: 'mockclone' });
+      expect(result).toEqual({
+        remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
+        catalogInfoUrl:
+          'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml',
+      });
       expect(mockGitApi.createRepository).toHaveBeenCalledWith(
         {
           name: 'repo',
@@ -61,7 +65,7 @@ describe('Azure Publisher', () => {
       );
       expect(pushToRemoteUserPass).toHaveBeenCalledWith(
         '/tmp/test',
-        'mockclone',
+        'https://dev.azure.com/organization/project/_git/repo',
         'notempty',
         'fake-token',
       );

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { PublisherBase } from './types';
+import { PublisherBase, PublisherOptions, PublisherResult } from './types';
 import { GitApi } from 'azure-devops-node-api/GitApi';
 import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces';
 import { pushToRemoteUserPass } from './helpers';
@@ -33,14 +33,12 @@ export class AzurePublisher implements PublisherBase {
   async publish({
     values,
     directory,
-  }: {
-    values: RequiredTemplateValues & Record<string, JsonValue>;
-    directory: string;
-  }): Promise<{ remoteUrl: string }> {
+  }: PublisherOptions): Promise<PublisherResult> {
     const remoteUrl = await this.createRemote(values);
     await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token);
+    const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`;
 
-    return { remoteUrl };
+    return { remoteUrl, catalogInfoUrl };
   }
 
   private async createRemote(

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts

@@ -53,7 +53,7 @@ describe('GitHub Publisher', () => {
       it('should use octokit to create a repo in an organisation if the organisation property is set', async () => {
         mockGithubClient.repos.createInOrg.mockResolvedValue({
           data: {
-            clone_url: 'mockclone',
+            clone_url: 'https://github.com/backstage/backstage.git',
           },
         } as OctokitResponse<ReposCreateInOrgResponseData>);
         mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -71,7 +71,11 @@ describe('GitHub Publisher', () => {
           directory: '/tmp/test',
         });
 
-        expect(result).toEqual({ remoteUrl: 'mockclone' });
+        expect(result).toEqual({
+          remoteUrl: 'https://github.com/backstage/backstage.git',
+          catalogInfoUrl:
+            'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+        });
         expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({
           org: 'blam',
           name: 'test',
@@ -89,7 +93,7 @@ describe('GitHub Publisher', () => {
         });
         expect(pushToRemoteUserPass).toHaveBeenCalledWith(
           '/tmp/test',
-          'mockclone',
+          'https://github.com/backstage/backstage.git',
           'abc',
           'x-oauth-basic',
         );
@@ -98,7 +102,7 @@ describe('GitHub Publisher', () => {
       it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => {
         mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
           data: {
-            clone_url: 'mockclone',
+            clone_url: 'https://github.com/backstage/backstage.git',
           },
         } as OctokitResponse<ReposCreateInOrgResponseData>);
         mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -116,7 +120,11 @@ describe('GitHub Publisher', () => {
           directory: '/tmp/test',
         });
 
-        expect(result).toEqual({ remoteUrl: 'mockclone' });
+        expect(result).toEqual({
+          remoteUrl: 'https://github.com/backstage/backstage.git',
+          catalogInfoUrl:
+            'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+        });
         expect(
           mockGithubClient.repos.createForAuthenticatedUser,
         ).toHaveBeenCalledWith({
@@ -126,7 +134,7 @@ describe('GitHub Publisher', () => {
         expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled();
         expect(pushToRemoteUserPass).toHaveBeenCalledWith(
           '/tmp/test',
-          'mockclone',
+          'https://github.com/backstage/backstage.git',
           'abc',
           'x-oauth-basic',
         );
@@ -136,7 +144,7 @@ describe('GitHub Publisher', () => {
     it('should invite other user in the authed user', async () => {
       mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
         data: {
-          clone_url: 'mockclone',
+          clone_url: 'https://github.com/backstage/backstage.git',
         },
       } as OctokitResponse<ReposCreateInOrgResponseData>);
       mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -155,7 +163,11 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
-      expect(result).toEqual({ remoteUrl: 'mockclone' });
+      expect(result).toEqual({
+        remoteUrl: 'https://github.com/backstage/backstage.git',
+        catalogInfoUrl:
+          'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+      });
       expect(
         mockGithubClient.repos.createForAuthenticatedUser,
       ).toHaveBeenCalledWith({
@@ -171,7 +183,7 @@ describe('GitHub Publisher', () => {
       });
       expect(pushToRemoteUserPass).toHaveBeenCalledWith(
         '/tmp/test',
-        'mockclone',
+        'https://github.com/backstage/backstage.git',
         'abc',
         'x-oauth-basic',
       );
@@ -188,7 +200,7 @@ describe('GitHub Publisher', () => {
     it('creates a private repository in the organization with visibility set to internal', async () => {
       mockGithubClient.repos.createInOrg.mockResolvedValue({
         data: {
-          clone_url: 'mockclone',
+          clone_url: 'https://github.com/backstage/backstage.git',
         },
       } as OctokitResponse<ReposCreateInOrgResponseData>);
       mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -206,7 +218,11 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
-      expect(result).toEqual({ remoteUrl: 'mockclone' });
+      expect(result).toEqual({
+        remoteUrl: 'https://github.com/backstage/backstage.git',
+        catalogInfoUrl:
+          'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+      });
       expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({
         org: 'blam',
         name: 'test',
@@ -215,7 +231,7 @@ describe('GitHub Publisher', () => {
       });
       expect(pushToRemoteUserPass).toHaveBeenCalledWith(
         '/tmp/test',
-        'mockclone',
+        'https://github.com/backstage/backstage.git',
         'abc',
         'x-oauth-basic',
       );
@@ -232,7 +248,7 @@ describe('GitHub Publisher', () => {
     it('creates a private repository', async () => {
       mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
         data: {
-          clone_url: 'mockclone',
+          clone_url: 'https://github.com/backstage/backstage.git',
         },
       } as OctokitResponse<ReposCreateInOrgResponseData>);
       mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -249,7 +265,11 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
-      expect(result).toEqual({ remoteUrl: 'mockclone' });
+      expect(result).toEqual({
+        remoteUrl: 'https://github.com/backstage/backstage.git',
+        catalogInfoUrl:
+          'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+      });
       expect(
         mockGithubClient.repos.createForAuthenticatedUser,
       ).toHaveBeenCalledWith({
@@ -258,7 +278,7 @@ describe('GitHub Publisher', () => {
       });
       expect(pushToRemoteUserPass).toHaveBeenCalledWith(
         '/tmp/test',
-        'mockclone',
+        'https://github.com/backstage/backstage.git',
         'abc',
         'x-oauth-basic',
       );

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { PublisherBase } from './types';
+import { PublisherBase, PublisherOptions, PublisherResult } from './types';
 import { Octokit } from '@octokit/rest';
 import { pushToRemoteUserPass } from './helpers';
 import { JsonValue } from '@backstage/config';
@@ -46,10 +46,7 @@ export class GithubPublisher implements PublisherBase {
   async publish({
     values,
     directory,
-  }: {
-    values: RequiredTemplateValues & Record<string, JsonValue>;
-    directory: string;
-  }): Promise<{ remoteUrl: string }> {
+  }: PublisherOptions): Promise<PublisherResult> {
     const remoteUrl = await this.createRemote(values);
     await pushToRemoteUserPass(
       directory,
@@ -57,8 +54,12 @@ export class GithubPublisher implements PublisherBase {
       this.token,
       'x-oauth-basic',
     );
+    const catalogInfoUrl = remoteUrl.replace(
+      /\.git$/,
+      '/blob/master/catalog-info.yaml',
+    );
 
-    return { remoteUrl };
+    return { remoteUrl, catalogInfoUrl };
   }
 
   private async createRemote(

plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { PublisherBase } from './types';
+import { PublisherBase, PublisherOptions, PublisherResult } from './types';
 import { Gitlab } from '@gitbeaker/core';
 import { pushToRemoteUserPass } from './helpers';
 import { JsonValue } from '@backstage/config';
@@ -32,10 +32,7 @@ export class GitlabPublisher implements PublisherBase {
   async publish({
     values,
     directory,
-  }: {
-    values: RequiredTemplateValues & Record<string, JsonValue>;
-    directory: string;
-  }): Promise<{ remoteUrl: string }> {
+  }: PublisherOptions): Promise<PublisherResult> {
     const remoteUrl = await this.createRemote(values);
     await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token);
 

plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts

@@ -29,11 +29,17 @@ export type PublisherBase = {
    *             catalog, plus the values from the form and the directory that has
    *             been templated
    */
-  publish(opts: {
-    entity: TemplateEntityV1alpha1;
-    values: RequiredTemplateValues & Record<string, JsonValue>;
-    directory: string;
-  }): Promise<{ remoteUrl: string }>;
+  publish(opts: PublisherOptions): Promise<PublisherResult>;
+};
+
+export type PublisherOptions = {
+  values: RequiredTemplateValues & Record<string, JsonValue>;
+  directory: string;
+};
+
+export type PublisherResult = {
+  remoteUrl: string;
+  catalogInfoUrl?: string;
 };
 
 export type PublisherBuilder = {

plugins/scaffolder-backend/src/service/router.ts

@@ -156,12 +156,11 @@ export async function createRouter(
             handler: async (ctx: StageContext<{ resultDir: string }>) => {
               const publisher = publishers.get(ctx.entity);
               ctx.logger.info('Will now store the template');
-              const { remoteUrl } = await publisher.publish({
-                entity: ctx.entity,
+              const result = await publisher.publish({
                 values: ctx.values,
                 directory: ctx.resultDir,
               });
-              return { remoteUrl };
+              return result;
             },
           },
         ],

plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx

@@ -30,18 +30,12 @@ import { JobStage } from '../JobStage/JobStage';
 import { useJobPolling } from './useJobPolling';
 
 type Props = {
-  onClose: () => void;
   onComplete: (job: Job) => void;
   jobId: string;
   entity: TemplateEntityV1alpha1 | null;
 };
 
-export const JobStatusModal = ({
-  onClose,
-  jobId,
-  onComplete,
-  entity,
-}: Props) => {
+export const JobStatusModal = ({ jobId, onComplete, entity }: Props) => {
   const job = useJobPolling(jobId);
   const [dialogTitle, setDialogTitle] = useState('Creating component...');
 
@@ -54,7 +48,7 @@ export const JobStatusModal = ({
   }, [job, onComplete, setDialogTitle]);
 
   return (
-    <Dialog open onClose={onClose} fullWidth>
+    <Dialog open fullWidth>
       <DialogTitle id="responsive-dialog-title">{dialogTitle}</DialogTitle>
       <DialogContent>
         {!job ? (

plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx

@@ -69,8 +69,9 @@ export const MultistepJsonForm = ({
         {steps.map(({ label, schema, ...formProps }) => (
           <Step key={label}>
             <StepLabel>{label}</StepLabel>
-            <StepContent>
+            <StepContent key={label}>
               <Form
+                key={label}
                 noHtml5Validate
                 formData={formData}
                 onChange={onChange}

plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx

@@ -26,7 +26,7 @@ import {
 import { catalogApiRef } from '@backstage/plugin-catalog';
 import { LinearProgress } from '@material-ui/core';
 import { IChangeEvent } from '@rjsf/core';
-import React, { useState } from 'react';
+import React, { useState, useCallback } from 'react';
 import { Navigate } from 'react-router';
 import { useParams } from 'react-router-dom';
 import { useAsync } from 'react-use';
@@ -86,11 +86,12 @@ export const TemplatePage = () => {
   const [formState, setFormState] = useState({});
 
   const handleFormReset = () => setFormState({});
-  const handleChange = (e: IChangeEvent) =>
-    setFormState({ ...formState, ...e.formData });
+  const handleChange = useCallback(
+    (e: IChangeEvent) => setFormState({ ...formState, ...e.formData }),
+    [setFormState, formState],
+  );
 
   const [jobId, setJobId] = useState<string | null>(null);
-  const handleClose = () => setJobId(null);
 
   const handleCreate = async () => {
     try {
@@ -106,18 +107,10 @@ export const TemplatePage = () => {
   );
 
   const handleCreateComplete = async (job: Job) => {
-    const target = job.metadata.remoteUrl?.replace(
-      /\.git$/,
-      // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we
-      //               want this to be more flexible, but before that we might want
-      //               to update all templates to use catalog-info.yaml instead.
-      '/blob/master/component-info.yaml',
-    );
-
-    if (!target) {
+    if (!job.metadata.catalogInfoUrl) {
       errorApi.post(
         new Error(
-          `Failed to find component-info.yaml file in ${job.metadata.remoteUrl}.`,
+          `Failed to find catalog-info.yaml file in ${job.metadata.remoteUrl}.`,
         ),
       );
       return;
@@ -125,7 +118,7 @@ export const TemplatePage = () => {
 
     const {
       entities: [createdEntity],
-    } = await catalogApi.addLocation({ target });
+    } = await catalogApi.addLocation({ target: job.metadata.catalogInfoUrl });
 
     setEntity((createdEntity as any) as TemplateEntityV1alpha1);
   };
@@ -161,7 +154,6 @@ export const TemplatePage = () => {
           <JobStatusModal
             onComplete={handleCreateComplete}
             jobId={jobId}
-            onClose={handleClose}
             entity={entity}
           />
         )}

plugins/scaffolder/src/types.ts

@@ -19,6 +19,7 @@ export type Job = {
     entity: any;
     values: any;
     remoteUrl?: string;
+    catalogInfoUrl?: string;
   };
   status: 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
   stages: Stage[];

plugins/search/package.json

@@ -21,14 +21,15 @@
   },
   "dependencies": {
     "@backstage/core": "^0.3.1",
+    "@backstage/plugin-catalog": "^0.2.2",
     "@backstage/theme": "^0.2.1",
     "@material-ui/core": "^4.11.0",
     "@material-ui/icons": "^4.9.1",
     "@material-ui/lab": "4.0.0-alpha.45",
-    "@backstage/plugin-catalog": "^0.2.2",
-    "react-router-dom": "6.0.0-beta.0",
+    "qs": "^6.9.4",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
+    "react-router-dom": "6.0.0-beta.0",
     "react-use": "^15.3.3"
   },
   "devDependencies": {
@@ -40,8 +41,8 @@
     "@testing-library/user-event": "^12.0.7",
     "@types/jest": "^26.0.7",
     "@types/node": "^12.0.0",
-    "msw": "^0.21.2",
-    "cross-fetch": "^3.0.6"
+    "cross-fetch": "^3.0.6",
+    "msw": "^0.21.2"
   },
   "files": [
     "dist"

plugins/search/src/components/SearchPage/SearchPage.tsx

@@ -13,22 +13,32 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import React, { useState } from 'react';
-
-import { Header, Content, Page } from '@backstage/core';
+import { Content, Header, Page, useQueryParamState } from '@backstage/core';
 import { Grid } from '@material-ui/core';
-
+import React, { useEffect, useState } from 'react';
+import { useDebounce } from 'react-use';
 import { SearchBar } from '../SearchBar';
 import { SearchResult } from '../SearchResult';
 
 export const SearchPage = () => {
-  const [searchQuery, setSearchQuery] = useState('');
+  const [queryString, setQueryString] = useQueryParamState<string>('query');
+  const [searchQuery, setSearchQuery] = useState(queryString ?? '');
 
   const handleSearch = (event: React.ChangeEvent<HTMLInputElement>) => {
     event.preventDefault();
     setSearchQuery(event.target.value);
   };
 
+  useEffect(() => setSearchQuery(queryString ?? ''), [queryString]);
+
+  useDebounce(
+    () => {
+      setQueryString(searchQuery);
+    },
+    200,
+    [searchQuery],
+  );
+
   const handleClearSearchBar = () => {
     setSearchQuery('');
   };
@@ -46,7 +56,7 @@ export const SearchPage = () => {
             />
           </Grid>
           <Grid item xs={12}>
-            <SearchResult searchQuery={searchQuery.toLowerCase()} />
+            <SearchResult searchQuery={(queryString ?? '').toLowerCase()} />
           </Grid>
         </Grid>
       </Content>

plugins/search/src/components/SearchResult/SearchResult.tsx

@@ -159,6 +159,9 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => {
         withFilters = withFilters.filter(
           (result: Result) =>
             result.name?.toLowerCase().includes(searchQuery) ||
+            result.name
+              ?.toLowerCase()
+              .includes(searchQuery.split(' ').join('-')) ||
             result.description?.toLowerCase().includes(searchQuery),
         );
       }

plugins/search/src/components/SidebarSearch/SidebarSearch.tsx

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React, { useCallback } from 'react';
+import qs from 'qs';
+import { useNavigate } from 'react-router-dom';
+import { SidebarSearchField } from '@backstage/core';
+
+export const SidebarSearch = () => {
+  const navigate = useNavigate();
+  const handleSearch = useCallback(
+    (query: string): void => {
+      const queryString = qs.stringify({ query }, { addQueryPrefix: true });
+
+      // TODO: Here the url to the search plugin is hardcoded. We need a way to query the route in the future.
+      // Maybe an API that I can just call from other places?
+      navigate(`/search${queryString}`);
+    },
+    [navigate],
+  );
+
+  return <SidebarSearchField onSearch={handleSearch} />;
+};

plugins/search/src/components/SidebarSearch/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { SidebarSearch } from './SidebarSearch';

plugins/search/src/components/index.tsx

@@ -18,3 +18,4 @@ export * from './Filters';
 export * from './SearchBar';
 export * from './SearchPage';
 export * from './SearchResult';
+export * from './SidebarSearch';

plugins/search/src/index.ts

@@ -14,3 +14,4 @@
  * limitations under the License.
  */
 export { plugin } from './plugin';
+export * from './components';

yarn.lock

@@ -87,6 +87,24 @@
   resolved "https://registry.npmjs.org/@asyncapi/specs/-/specs-2.7.5.tgz#3a516d198fc41a1103695bd889fdd4fbbebe7f5d"
   integrity sha512-T1Ham9sqZKCtSowXRPaBCRy2oz3KHglqqrKiaO7lEudpP6lwH5SwXaq4qliyKzWaqd22srJHE4szdsorbFZKlw==
 
+"@azure/msal-common@^1.6.2":
+  version "1.6.2"
+  resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-1.6.2.tgz#91f3732866d727e20f1e142e6e88a981268fbff2"
+  integrity sha512-GShzp1q7Ld8SwYiDEjQZ9PmFOY4x+2stE86maiguylE9/d/c2muqKjc8aepmEqyjbV7o/omDvEf2Sr9QcIqkSA==
+  dependencies:
+    debug "^4.1.1"
+
+"@azure/msal-node@^1.0.0-alpha.8":
+  version "1.0.0-alpha.12"
+  resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-alpha.12.tgz#09d8d52f5cea90b133c3d48fe4ec477693040c91"
+  integrity sha512-uGLOJRWiEhfJIrTv/lwdm4RxQFm++00h83zNgDn0O3NkXlzAoCCq9QFYW84PjMR/Q2PUvVy7uW+6yKL/Nq3gBA==
+  dependencies:
+    "@azure/msal-common" "^1.6.2"
+    axios "^0.19.2"
+    debug "^4.1.1"
+    jsonwebtoken "^8.5.1"
+    uuid "^8.3.0"
+
 "@babel/code-frame@7.0.0":
   version "7.0.0"
   resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
@@ -1289,42 +1307,6 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
-"@backstage/core@^0.2.0":
-  version "0.3.1"
-  dependencies:
-    "@backstage/config" "^0.1.1"
-    "@backstage/core-api" "^0.2.1"
-    "@backstage/theme" "^0.2.1"
-    "@material-ui/core" "^4.11.0"
-    "@material-ui/icons" "^4.9.1"
-    "@material-ui/lab" "4.0.0-alpha.45"
-    "@types/dagre" "^0.7.44"
-    "@types/react" "^16.9"
-    "@types/react-sparklines" "^1.7.0"
-    classnames "^2.2.6"
-    clsx "^1.1.0"
-    d3-selection "^2.0.0"
-    d3-shape "^2.0.0"
-    d3-zoom "^2.0.0"
-    dagre "^0.8.5"
-    immer "^7.0.9"
-    lodash "^4.17.15"
-    material-table "^1.69.1"
-    prop-types "^15.7.2"
-    qs "^6.9.4"
-    rc-progress "^3.0.0"
-    react "^16.12.0"
-    react-dom "^16.12.0"
-    react-helmet "6.1.0"
-    react-hook-form "^6.6.0"
-    react-markdown "^5.0.2"
-    react-router "6.0.0-beta.0"
-    react-router-dom "6.0.0-beta.0"
-    react-sparklines "^1.7.0"
-    react-syntax-highlighter "^13.5.1"
-    react-use "^15.3.3"
-    remark-gfm "^1.0.0"
-
 "@backstage/plugin-catalog-backend@^0.2.1":
   version "0.2.1"
   resolved "https://registry.npmjs.org/@backstage/plugin-catalog-backend/-/plugin-catalog-backend-0.2.1.tgz#d63d1d145d97a5f1021c3e6c2ab680729be43e74"
@@ -3401,6 +3383,11 @@
   resolved "https://registry.npmjs.org/@mdx-js/react/-/react-1.5.9.tgz#31873ab097fbe58c61c7585fc0be64e83182b6df"
   integrity sha512-rengdUSedIdIQbXPSeafItCacTYocARAjUA51b6R1KNHmz+59efz7UmyTKr73viJQZ98ouu7iRGmOTtjRrbbWA==
 
+"@microsoft/microsoft-graph-types@^1.25.0":
+  version "1.25.0"
+  resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-1.25.0.tgz#1f543ebc029a115dd1d48a1ae99d7ddd5ee9af57"
+  integrity sha512-RsuA+ROaU3voWzG9TVBkRKxmLatteRGduFDi5p0k3FUHho49rm9SvrA7DUyYbSXLy2xXRx9AnjKM9klYBeKEiQ==
+
 "@mrmlnc/readdir-enhanced@^2.2.1":
   version "2.2.1"
   resolved "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde"
@@ -3741,14 +3728,14 @@
   resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.0.tgz#1b5859298bf3f61137d7b05084f058a775d6fd73"
   integrity sha512-U8F/suzg4MuV+8mK1/ufs0Y6c3O8hc1wnuD2IKoOVJvegGfz5JCafyoyGAW6iyuT1DZBMPzVWEqfiuYPmoE7pw==
 
-"@roadiehq/backstage-plugin-buildkite@^0.1.2":
-  version "0.1.2"
-  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-buildkite/-/backstage-plugin-buildkite-0.1.2.tgz#2f2b414acc18ed7820abc6fb0c042f00c31d7c9a"
-  integrity sha512-NO1ogAK6lfh/YUmftqbYn6pkUmMJXk9cfda0YDnG23XlcKIst1d3xO5FggozVtU5IZiJDxxwxOOo4ylhBWrGeQ==
+"@roadiehq/backstage-plugin-buildkite@^0.1.3":
+  version "0.1.3"
+  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-buildkite/-/backstage-plugin-buildkite-0.1.3.tgz#5a116bf677dfad22088212dde398cf3e9b48d6e4"
+  integrity sha512-q+cnAvZmjLu0DcqRKJZJhWTVcKaA9j7cM44tx6JcshEyb3UgzrlECLvx4ethnYyrmhM1/FSpMLa+t3N5A1Xz4g==
   dependencies:
     "@backstage/catalog-model" "^0.2.0"
-    "@backstage/core" "^0.2.0"
-    "@backstage/plugin-catalog" "^0.2.0"
+    "@backstage/core" "^0.3.0"
+    "@backstage/plugin-catalog" "^0.2.1"
     "@material-ui/core" "^4.11.0"
     "@material-ui/icons" "^4.9.1"
     "@material-ui/lab" "4.0.0-alpha.45"
@@ -3760,14 +3747,14 @@
     react-router-dom "6.0.0-beta.0"
     react-use "^15.3.3"
 
-"@roadiehq/backstage-plugin-github-insights@^0.2.14":
-  version "0.2.14"
-  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.14.tgz#2e4bf61495e650bb2b7a1711f97c9c4995215588"
-  integrity sha512-xhzHAmmTu7op1V7K3ytomGlmzHMs9jnb2Lc2YTrU3ame4WZcrrMCrqi8X9v8B8VHfwMBrwa55tyxRYObXrPr7A==
+"@roadiehq/backstage-plugin-github-insights@^0.2.15":
+  version "0.2.15"
+  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.15.tgz#b127e7795d3a2440286548fc33af49f12ba42b7f"
+  integrity sha512-4jLDHlr5B77Kveb7Q1q78mbMATufJLgS/zy4T+1LW/54KKmJrset4AWrZLTAOwjkjOfDfO85NEwmgsbrprAGrg==
   dependencies:
     "@backstage/catalog-model" "^0.2.0"
-    "@backstage/core" "^0.2.0"
-    "@backstage/theme" "^0.2.0"
+    "@backstage/core" "^0.3.0"
+    "@backstage/theme" "^0.2.1"
     "@material-ui/core" "^4.11.0"
     "@material-ui/icons" "^4.9.1"
     "@material-ui/lab" "4.0.0-alpha.45"
@@ -3779,13 +3766,13 @@
     react-router "^6.0.0-beta.0"
     react-use "^15.3.3"
 
-"@roadiehq/backstage-plugin-github-pull-requests@^0.6.2":
-  version "0.6.2"
-  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.2.tgz#02f4a7a03e1a7dc24342ec695130017ce7d3ad8d"
-  integrity sha512-OdBWO6NdiBKlol1WlbFQGeHxV2C8wl8EiI2NgA15lozsqFA22kWis6Z16cAHe4ZdRCjihDmqINtAIJzeZZ1gOg==
+"@roadiehq/backstage-plugin-github-pull-requests@^0.6.3":
+  version "0.6.3"
+  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.3.tgz#46b63e90f3f5412a4b8f0df96ada62cff7046478"
+  integrity sha512-ofWH9k4WVVwTbK/XAVqmtH03QW3rsT822p4neOse0Wy0dAKlqlSU4nwl5jBKMJXsf8lfc0c7gbc50R7VumzoOQ==
   dependencies:
     "@backstage/catalog-model" "^0.2.0"
-    "@backstage/core" "^0.2.0"
+    "@backstage/core" "^0.3.0"
     "@backstage/plugin-catalog" "^0.2.0"
     "@backstage/theme" "^0.2.0"
     "@material-ui/core" "^4.11.0"
@@ -3801,16 +3788,16 @@
     react-router "6.0.0-beta.0"
     react-use "^15.3.3"
 
-"@roadiehq/backstage-plugin-travis-ci@^0.2.7":
-  version "0.2.7"
-  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-travis-ci/-/backstage-plugin-travis-ci-0.2.7.tgz#bc7968b461016b2710794d10766266de9ab4f759"
-  integrity sha512-uXF5t2uZqd9TNGFSMYlyk6NAweVb5KLlM4GMiltattzxRWiqbCp5VumubrLTTsA8dvsWCrXWfhe95MStcUnl+A==
+"@roadiehq/backstage-plugin-travis-ci@^0.2.8":
+  version "0.2.8"
+  resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-travis-ci/-/backstage-plugin-travis-ci-0.2.8.tgz#c730531519bf3e3dba35522d5e6c822176ddb9c8"
+  integrity sha512-8R8waHoviT5chHGybAMVVRHHmeytn+8Po7sU08WCYIt4Vkw2/gwu36fd90y+lW4x6T0s3ZiaJrsQ3KlA7LmRnQ==
   dependencies:
     "@backstage/catalog-model" "^0.2.0"
-    "@backstage/core" "^0.2.0"
-    "@backstage/core-api" "^0.2.0"
-    "@backstage/plugin-catalog" "^0.2.0"
-    "@backstage/theme" "^0.2.0"
+    "@backstage/core" "^0.3.0"
+    "@backstage/core-api" "^0.2.1"
+    "@backstage/plugin-catalog" "^0.2.1"
+    "@backstage/theme" "^0.2.1"
     "@material-ui/core" "^4.9.1"
     "@material-ui/icons" "^4.9.1"
     "@material-ui/lab" "4.0.0-alpha.45"
@@ -4904,6 +4891,13 @@
   resolved "https://registry.npmjs.org/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a"
   integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA==
 
+"@types/archiver@^3.1.1":
+  version "3.1.1"
+  resolved "https://registry.npmjs.org/@types/archiver/-/archiver-3.1.1.tgz#10cc1be44af8911e57484342c7b3b32a5f178a1a"
+  integrity sha512-TzVZ9204sH1TuFylfr1cw/AA/3/VldAAXswEwKLXUOzA9mDg+m6gHF9EaqKNlozcjc6knX5m1KAqJzksPLSEfw==
+  dependencies:
+    "@types/glob" "*"
+
 "@types/aria-query@^4.2.0":
   version "4.2.0"
   resolved "https://registry.npmjs.org/@types/aria-query/-/aria-query-4.2.0.tgz#14264692a9d6e2fa4db3df5e56e94b5e25647ac0"
@@ -6047,6 +6041,13 @@
   resolved "https://registry.npmjs.org/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
   integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==
 
+"@types/unzipper@^0.10.3":
+  version "0.10.3"
+  resolved "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.3.tgz#9eea872fb1fa460da76f253878b6275af588f464"
+  integrity sha512-01mQdTLp3/KuBVDhP82FNBf+enzVOjJ9dGsCWa5z8fcYAFVgA9bqIQ2NmsgNFzN/DhD0PUQj4n5p7k6I9mq80g==
+  dependencies:
+    "@types/node" "*"
+
 "@types/uuid@^8.0.0":
   version "8.0.0"
   resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.0.0.tgz#165aae4819ad2174a17476dbe66feebd549556c0"
@@ -6958,6 +6959,35 @@ arch@^2.1.2:
   resolved "https://registry.npmjs.org/arch/-/arch-2.1.2.tgz#0c52bbe7344bb4fa260c443d2cbad9c00ff2f0bf"
   integrity sha512-NTBIIbAfkJeIletyABbVtdPgeKfDafR+1mZV/AyyfC1UkVkp9iUjV+wwmqtUgphHYajbI86jejBJp5e+jkGTiQ==
 
+archiver-utils@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2"
+  integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==
+  dependencies:
+    glob "^7.1.4"
+    graceful-fs "^4.2.0"
+    lazystream "^1.0.0"
+    lodash.defaults "^4.2.0"
+    lodash.difference "^4.5.0"
+    lodash.flatten "^4.4.0"
+    lodash.isplainobject "^4.0.6"
+    lodash.union "^4.6.0"
+    normalize-path "^3.0.0"
+    readable-stream "^2.0.0"
+
+archiver@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.npmjs.org/archiver/-/archiver-5.0.2.tgz#b2c435823499b1f46eb07aa18e7bcb332f6ca3fc"
+  integrity sha512-Tq3yV/T4wxBsD2Wign8W9VQKhaUxzzRmjEiSoOK0SLqPgDP/N1TKdYyBeIEu56T4I9iO4fKTTR0mN9NWkBA0sg==
+  dependencies:
+    archiver-utils "^2.1.0"
+    async "^3.2.0"
+    buffer-crc32 "^0.2.1"
+    readable-stream "^3.6.0"
+    readdir-glob "^1.0.0"
+    tar-stream "^2.1.4"
+    zip-stream "^4.0.0"
+
 are-we-there-yet@~1.1.2:
   version "1.1.5"
   resolved "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21"
@@ -7711,6 +7741,11 @@ base64-js@^1.0.2, base64-js@^1.2.0:
   resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1"
   integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==
 
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 base64url@3.x.x, base64url@^3.0.1:
   version "3.0.1"
   resolved "https://registry.npmjs.org/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
@@ -7782,6 +7817,11 @@ bfj@^7.0.2:
     hoopy "^0.1.4"
     tryer "^1.0.1"
 
+big-integer@^1.6.17:
+  version "1.6.48"
+  resolved "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e"
+  integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==
+
 big.js@^5.2.2:
   version "5.2.2"
   resolved "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"
@@ -7797,6 +7837,14 @@ binary-extensions@^2.0.0:
   resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
   integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
 
+binary@~0.3.0:
+  version "0.3.0"
+  resolved "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79"
+  integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=
+  dependencies:
+    buffers "~0.1.1"
+    chainsaw "~0.1.0"
+
 bindings@^1.5.0:
   version "1.5.0"
   resolved "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
@@ -7817,7 +7865,7 @@ bl@^1.0.0:
     readable-stream "^2.3.5"
     safe-buffer "^5.1.1"
 
-bl@^4.0.1:
+bl@^4.0.1, bl@^4.0.3:
   version "4.0.3"
   resolved "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489"
   integrity sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==
@@ -7838,6 +7886,11 @@ bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.5
   resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
+bluebird@~3.4.1:
+  version "3.4.7"
+  resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3"
+  integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.4.0:
   version "4.11.8"
   resolved "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"
@@ -8072,7 +8125,7 @@ buffer-alloc@^1.2.0:
     buffer-alloc-unsafe "^1.1.0"
     buffer-fill "^1.0.0"
 
-buffer-crc32@~0.2.3:
+buffer-crc32@^0.2.1, buffer-crc32@^0.2.13, buffer-crc32@~0.2.3:
   version "0.2.13"
   resolved "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
   integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=
@@ -8092,6 +8145,11 @@ buffer-from@1.x, buffer-from@^1.0.0:
   resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer-indexof-polyfill@~1.0.0:
+  version "1.0.2"
+  resolved "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz#d2732135c5999c64b277fcf9b1abe3498254729c"
+  integrity sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==
+
 buffer-indexof@^1.0.0:
   version "1.1.1"
   resolved "https://registry.npmjs.org/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c"
@@ -8121,6 +8179,14 @@ buffer@^4.3.0:
     ieee754 "^1.1.4"
     isarray "^1.0.0"
 
+buffer@^5.1.0:
+  version "5.7.1"
+  resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
+  integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.1.13"
+
 buffer@^5.5.0, buffer@^5.6.0:
   version "5.6.0"
   resolved "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
@@ -8129,6 +8195,11 @@ buffer@^5.5.0, buffer@^5.6.0:
     base64-js "^1.0.2"
     ieee754 "^1.1.4"
 
+buffers@~0.1.1:
+  version "0.1.1"
+  resolved "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb"
+  integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s=
+
 bufferutil@^4.0.1:
   version "4.0.1"
   resolved "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.1.tgz#3a177e8e5819a1243fe16b63a199951a7ad8d4a7"
@@ -8422,6 +8493,13 @@ caseless@~0.12.0:
   resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
   integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
 
+chainsaw@~0.1.0:
+  version "0.1.0"
+  resolved "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98"
+  integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=
+  dependencies:
+    traverse ">=0.3.0 <0.4"
+
 chalk@2.4.2, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.1, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -8964,6 +9042,16 @@ component-emitter@^1.2.0, component-emitter@^1.2.1:
   resolved "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
   integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
 
+compress-commons@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.npmjs.org/compress-commons/-/compress-commons-4.0.1.tgz#c5fa908a791a0c71329fba211d73cd2a32005ea8"
+  integrity sha512-xZm9o6iikekkI0GnXCmAl3LQGZj5TBDj0zLowsqi7tJtEa3FMGSEcHcqrSJIrOAk1UG/NBbDn/F1q+MG/p/EsA==
+  dependencies:
+    buffer-crc32 "^0.2.13"
+    crc32-stream "^4.0.0"
+    normalize-path "^3.0.0"
+    readable-stream "^3.6.0"
+
 compressible@~2.0.16:
   version "2.0.18"
   resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba"
@@ -9331,6 +9419,21 @@ cosmiconfig@^7.0.0:
     path-type "^4.0.0"
     yaml "^1.10.0"
 
+crc32-stream@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.0.tgz#05b7ca047d831e98c215538666f372b756d91893"
+  integrity sha512-tyMw2IeUX6t9jhgXI6um0eKfWq4EIDpfv5m7GX4Jzp7eVelQ360xd8EPXJhp2mHwLQIkqlnMLjzqSZI3a+0wRw==
+  dependencies:
+    crc "^3.4.4"
+    readable-stream "^3.4.0"
+
+crc@^3.4.4:
+  version "3.8.0"
+  resolved "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6"
+  integrity sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==
+  dependencies:
+    buffer "^5.1.0"
+
 create-ecdh@^4.0.0:
   version "4.0.3"
   resolved "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"
@@ -10654,6 +10757,13 @@ dotenv@^8.0.0, dotenv@^8.2.0:
   resolved "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
   integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
 
+duplexer2@~0.1.4:
+  version "0.1.4"
+  resolved "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
+  integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=
+  dependencies:
+    readable-stream "^2.0.2"
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -15445,6 +15555,13 @@ lazy-universal-dotenv@^3.0.1:
     dotenv "^8.0.0"
     dotenv-expand "^5.1.0"
 
+lazystream@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4"
+  integrity sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=
+  dependencies:
+    readable-stream "^2.0.5"
+
 lcid@^1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835"
@@ -15589,6 +15706,11 @@ lint-staged@^10.1.0:
     string-argv "0.3.1"
     stringify-object "^3.3.0"
 
+listenercount@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937"
+  integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=
+
 listr-silent-renderer@^1.1.1:
   version "1.1.1"
   resolved "https://registry.npmjs.org/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e"
@@ -15791,6 +15913,16 @@ lodash.debounce@^4, lodash.debounce@^4.0.8:
   resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
   integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168=
 
+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
+
+lodash.difference@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
+  integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=
+
 lodash.flatten@^4.4.0:
   version "4.4.0"
   resolved "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
@@ -15886,6 +16018,11 @@ lodash.throttle@^4.1.1:
   resolved "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4"
   integrity sha1-wj6RtxAkKscMN/HhzaknTMOb8vQ=
 
+lodash.union@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
+  integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
+
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@@ -20236,7 +20373,7 @@ read@1, read@~1.0.1:
   dependencies:
     mute-stream "~0.0.4"
 
-"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6:
+"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6:
   version "2.3.7"
   resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -20249,7 +20386,7 @@ read@1, read@~1.0.1:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-"readable-stream@2 || 3", readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0:
+"readable-stream@2 || 3", readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -20258,6 +20395,13 @@ read@1, read@~1.0.1:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
+readdir-glob@^1.0.0:
+  version "1.1.1"
+  resolved "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
+  integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==
+  dependencies:
+    minimatch "^3.0.4"
+
 readdir-scoped-modules@^1.0.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@@ -21247,7 +21391,7 @@ set-value@^2.0.0, set-value@^2.0.1:
     is-plain-object "^2.0.3"
     split-string "^3.0.1"
 
-setimmediate@^1.0.4, setimmediate@^1.0.5:
+setimmediate@^1.0.4, setimmediate@^1.0.5, setimmediate@~1.0.4:
   version "1.0.5"
   resolved "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
   integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=
@@ -22502,6 +22646,17 @@ tar-stream@^2.0.0:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+tar-stream@^2.1.4:
+  version "2.1.4"
+  resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz#c4fb1a11eb0da29b893a5b25476397ba2d053bfa"
+  integrity sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==
+  dependencies:
+    bl "^4.0.3"
+    end-of-stream "^1.4.1"
+    fs-constants "^1.0.0"
+    inherits "^2.0.3"
+    readable-stream "^3.1.1"
+
 tar@^2.0.0:
   version "2.2.2"
   resolved "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40"
@@ -22936,6 +23091,11 @@ tr46@^2.0.2:
   dependencies:
     punycode "^2.1.1"
 
+"traverse@>=0.3.0 <0.4":
+  version "0.3.9"
+  resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9"
+  integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=
+
 traverse@~0.6.6:
   version "0.6.6"
   resolved "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz#cbdf560fd7b9af632502fed40f918c157ea97137"
@@ -23457,6 +23617,22 @@ untildify@^4.0.0:
   resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
   integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==
 
+unzipper@^0.10.11:
+  version "0.10.11"
+  resolved "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz#0b4991446472cbdb92ee7403909f26c2419c782e"
+  integrity sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==
+  dependencies:
+    big-integer "^1.6.17"
+    binary "~0.3.0"
+    bluebird "~3.4.1"
+    buffer-indexof-polyfill "~1.0.0"
+    duplexer2 "~0.1.4"
+    fstream "^1.0.12"
+    graceful-fs "^4.2.2"
+    listenercount "~1.0.1"
+    readable-stream "~2.3.6"
+    setimmediate "~1.0.4"
+
 upath@^1.1.1, upath@^1.2.0:
   version "1.2.0"
   resolved "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"
@@ -24655,6 +24831,15 @@ zenscroll@^4.0.2:
   resolved "https://registry.npmjs.org/zenscroll/-/zenscroll-4.0.2.tgz#e8d5774d1c0738a47bcfa8729f3712e2deddeb25"
   integrity sha1-6NV3TRwHOKR7z6hynzcS4t7d6yU=
 
+zip-stream@^4.0.0:
+  version "4.0.2"
+  resolved "https://registry.npmjs.org/zip-stream/-/zip-stream-4.0.2.tgz#3a20f1bd7729c2b59fd4efa04df5eb7a5a217d2e"
+  integrity sha512-TGxB2g+1ur6MHkvM644DuZr8Uzyz0k0OYWtS3YlpfWBEmK4woaC2t3+pozEL3dBfIPmpgmClR5B2QRcMgGt22g==
+  dependencies:
+    archiver-utils "^2.1.0"
+    compress-commons "^4.0.0"
+    readable-stream "^3.6.0"
+
 zombie@^6.1.4:
   version "6.1.4"
   resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47"