diff --git a/.changeset/loud-vans-greet.md b/.changeset/loud-vans-greet.md
new file mode 100644
index 0000000000..732413174b
--- /dev/null
+++ b/.changeset/loud-vans-greet.md
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+Internal refactor to only create one external token handler
diff --git a/packages/backend-app-api/src/services/implementations/auth/authServiceFactory.ts b/packages/backend-app-api/src/services/implementations/auth/authServiceFactory.ts
index 462aabfa21..7bbd012c01 100644
--- a/packages/backend-app-api/src/services/implementations/auth/authServiceFactory.ts
+++ b/packages/backend-app-api/src/services/implementations/auth/authServiceFactory.ts
@@ -39,8 +39,19 @@ export const authServiceFactory = createServiceFactory({
     // new auth services in the new backend system.
     tokenManager: coreServices.tokenManager,
   },
-
-  async factory({ config, discovery, plugin, tokenManager, logger, database }) {
+  async createRootContext({ config, logger }) {
+    const externalTokens = ExternalTokenHandler.create({
+      config,
+      logger,
+    });
+    return {
+      externalTokens,
+    };
+  },
+  async factory(
+    { config, discovery, plugin, tokenManager, logger, database },
+    { externalTokens },
+  ) {
     const disableDefaultAuthPolicy = Boolean(
       config.getOptionalBoolean(
         'backend.auth.dangerouslyDisableDefaultAuthPolicy',
@@ -57,15 +68,11 @@ export const authServiceFactory = createServiceFactory({
     });
     const pluginTokens = PluginTokenHandler.create({
       ownPluginId: plugin.getId(),
-      keyDurationSeconds: 60 * 60,
+      keyDuration: { hours: 1 },
       logger,
       publicKeyStore,
       discovery,
     });
-    const externalTokens = ExternalTokenHandler.create({
-      config,
-      logger,
-    });
 
     return new DefaultAuthService(
       userTokens,
diff --git a/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.test.ts b/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.test.ts
index 2f23203768..424e01660f 100644
--- a/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.test.ts
+++ b/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.test.ts
@@ -31,7 +31,7 @@ describe('PluginTokenHandler', () => {
     const addKeyMock = jest.fn();
     const handler = PluginTokenHandler.create({
       discovery: mockServices.discovery(),
-      keyDurationSeconds: 10,
+      keyDuration: { seconds: 10 },
       logger: mockServices.logger.mock(),
       ownPluginId: 'test',
       publicKeyStore: {
diff --git a/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.ts b/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.ts
index 85cfa26285..fef33907a5 100644
--- a/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.ts
+++ b/packages/backend-app-api/src/services/implementations/auth/plugin/PluginTokenHandler.ts
@@ -30,6 +30,7 @@ import { AuthenticationError } from '@backstage/errors';
 import { jwtVerify } from 'jose';
 import { tokenTypes } from '@backstage/plugin-auth-node';
 import { JwksClient } from '../JwksClient';
+import { HumanDuration, durationToMilliseconds } from '@backstage/types';
 
 /**
  * The margin for how many times longer we make the public key available
@@ -45,8 +46,8 @@ type Options = {
   publicKeyStore: KeyStore;
   discovery: DiscoveryService;
   logger: LoggerService;
-  /** Expiration time of signing keys in seconds */
-  keyDurationSeconds: number;
+  /** Expiration time of signing keys */
+  keyDuration: HumanDuration;
   /** JWS "alg" (Algorithm) Header Parameter value. Defaults to ES256.
    * Must match one of the algorithms defined for IdentityClient.
    * When setting a different algorithm, check if the `key` field
@@ -70,7 +71,7 @@ export class PluginTokenHandler {
       options.logger,
       options.ownPluginId,
       options.publicKeyStore,
-      options.keyDurationSeconds,
+      Math.round(durationToMilliseconds(options.keyDuration) / 1000),
       options.algorithm ?? 'ES256',
       options.discovery,
     );