TechDocs: Implement etag based caching for url preparer

Signed-off-by: Himanshu Mishra <himanshu@orkohunter.net>

packages/techdocs-common/src/helpers.test.ts

@@ -139,7 +139,7 @@ describe('getDocFilesFromRepository', () => {
           archive: async () => {
             return Readable.from('');
           },
-          etag: '',
+          etag: 'etag123abc',
         };
       }
 
@@ -156,6 +156,7 @@ describe('getDocFilesFromRepository', () => {
       mockEntityWithAnnotation,
     );
 
-    expect(output).toBe('/tmp/testfolder');
+    expect(output.preparedDir).toBe('/tmp/testfolder');
+    expect(output.etag).toBe('etag123abc');
   });
 });

packages/techdocs-common/src/helpers.ts

@@ -14,17 +14,17 @@
  * limitations under the License.
  */
 
-import os from 'os';
-import path from 'path';
-import parseGitUrl from 'git-url-parse';
-import fs from 'fs-extra';
-import { InputError, UrlReader, Git } from '@backstage/backend-common';
+import { Git, InputError, UrlReader } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
+import fs from 'fs-extra';
+import parseGitUrl from 'git-url-parse';
+import os from 'os';
+import path from 'path';
+import { Logger } from 'winston';
 import { getDefaultBranch } from './default-branch';
 import { getGitRepoType, getTokenForGitRepo } from './git-auth';
-import { RemoteProtocol } from './stages/prepare/types';
-import { Logger } from 'winston';
+import { PreparerResponse, RemoteProtocol } from './stages/prepare/types';
 
 export type ParsedLocationAnnotation = {
   type: RemoteProtocol;
@@ -216,13 +216,19 @@ export const getLastCommitTimestamp = async (
 export const getDocFilesFromRepository = async (
   reader: UrlReader,
   entity: Entity,
-): Promise<any> => {
+  opts?: { etag?: string },
+): Promise<PreparerResponse> => {
   const { target } = parseReferenceAnnotation(
     'backstage.io/techdocs-ref',
     entity,
   );
 
-  const response = await reader.readTree(target);
+  // readTree will throw NotModifiedError if etag has not changed.
+  const readTreeResponse = await reader.readTree(target, { etag: opts?.etag });
+  const preparedDir = await readTreeResponse.dir();
 
-  return await response.dir();
+  return {
+    preparedDir,
+    etag: readTreeResponse.etag,
+  };
 };

packages/techdocs-common/src/stages/prepare/types.ts

@@ -33,13 +33,13 @@ export type PreparerBase = {
    * with contents from the location in temporary storage and return the path.
    *
    * @param entity The entity from the Service Catalog
-   * @param opts.etag (Optional) If etag is provider, it will be used to check if the target has
+   * @param options.etag (Optional) If etag is provider, it will be used to check if the target has
    * updated since the last build.
    * @throws {NotModifiedError} when the prepared directory has not been changed since the last build.
    */
   prepare(
     entity: Entity,
-    opts?: { logger: Logger; etag?: string },
+    options?: { logger?: Logger; etag?: string },
   ): Promise<PreparerResponse>;
 };
 

packages/techdocs-common/src/stages/prepare/url.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { UrlReader } from '@backstage/backend-common';
+import { NotModifiedError, UrlReader } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Logger } from 'winston';
 import { getDocFilesFromRepository } from '../../helpers';
@@ -28,19 +28,24 @@ export class UrlPreparer implements PreparerBase {
     this.reader = reader;
   }
 
-  async prepare(entity: Entity): Promise<PreparerResponse> {
+  async prepare(
+    entity: Entity,
+    options?: { etag?: string },
+  ): Promise<PreparerResponse> {
     try {
-      const preparedDir = await getDocFilesFromRepository(this.reader, entity);
-      // TODO: This will be actual etag from URL Reader.
-      const etag = '';
-      return {
-        preparedDir,
-        etag,
-      };
+      return await getDocFilesFromRepository(this.reader, entity, {
+        etag: options?.etag,
+      });
     } catch (error) {
-      this.logger.debug(
-        `Unable to fetch files for building docs ${error.message}`,
-      );
+      // NotModifiedError means that etag based cache is still valid.
+      if (error instanceof NotModifiedError) {
+        this.logger.debug(`Cache is valid for etag ${options?.etag}`);
+      } else {
+        this.logger.debug(
+          `Unable to fetch files for building docs ${error.message}`,
+        );
+      }
+
       throw error;
     }
   }

packages/techdocs-common/src/stages/publish/local.ts

@@ -88,7 +88,7 @@ export class LocalPublish implements PublisherBase {
           );
           reject(err);
         }
-
+        this.logger.info(`Published site stored at ${publishDir}`);
         this.discovery
           .getBaseUrl('techdocs')
           .then(techdocsApiUrl => {