Experimenting with giving developers the ability to choose their database with Humanitec

app-config.yaml

@@ -81,8 +81,8 @@ catalog:
   locations:
     - type: url
       target: https://github.com/thefrontside/backstage/blob/main/catalog-info.yaml
-    - type: url
-      target: https://github.com/thefrontside/backstage/blob/main/templates/standard-microservice/template.yaml
+    - type: file
+      target: ../../templates/standard-microservice/template.yaml
       rules:
         - allow: [Template]
 

packages/app/package.json

@@ -27,12 +27,13 @@
     "@backstage/plugin-search-react": "^0.2.1",
     "@backstage/plugin-tech-radar": "^0.5.11",
     "@backstage/plugin-techdocs": "^1.1.0",
-    "@backstage/plugin-techdocs-react": "^1.0.1",
     "@backstage/plugin-techdocs-module-addons-contrib": "^1.0.1",
+    "@backstage/plugin-techdocs-react": "^1.0.1",
     "@backstage/plugin-user-settings": "^0.4.3",
     "@backstage/theme": "^0.2.15",
     "@frontside/backstage-plugin-effection-inspector": "^0.1.0",
     "@frontside/backstage-plugin-humanitec": "^0.3.0",
+    "@frontside/backstage-plugin-platform": "^0.1.0",
     "@material-ui/core": "^4.12.2",
     "@material-ui/icons": "^4.9.1",
     "history": "^5.0.0",

packages/app/src/App.tsx

@@ -33,6 +33,7 @@ import { FlatRoutes } from '@backstage/core-app-api';
 import { orgPlugin } from '@backstage/plugin-org';
 import { InspectorPage } from '@frontside/backstage-plugin-effection-inspector';
 import { GraphiQLPage } from '@backstage/plugin-graphiql';
+import { PlatformPage } from '@frontside/backstage-plugin-platform';
 
 const app = createApp({
   apis,
@@ -90,6 +91,7 @@ const routes = (
     <Route path="/settings" element={<UserSettingsPage />} />
     <Route path="/effection-inspector" element={<InspectorPage />} />
     <Route path="/graphiql" element={<GraphiQLPage />} />
+    <Route path="/platform" element={<PlatformPage />}/>
   </FlatRoutes>
 );
 

packages/backend/.eslintrc.js

@@ -1 +1,5 @@
-module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, {
+  rules: {
+    'prefer-const': 'off'
+  }
+});

packages/backend/.gitignore

@@ -0,0 +1 @@
+/dist-bin/

packages/backend/package.json

@@ -38,10 +38,12 @@
     "@backstage/plugin-techdocs-backend": "^1.1.0",
     "@frontside/backstage-plugin-effection-inspector-backend": "0.1.1",
     "@frontside/backstage-plugin-batch-loader": "0.2.1",
+    "@frontside/backstage-plugin-humanitec-common": "*",
     "@frontside/backstage-plugin-humanitec-backend": "^0.3.0",
     "@frontside/backstage-plugin-graphql": "^0.4.0",
     "@frontside/backstage-plugin-incremental-ingestion-backend": "*",
     "@frontside/backstage-plugin-incremental-ingestion-github": "*",
+    "@frontside/backstage-plugin-platform-backend": "*",
     "graphql-modules": "^2.1.0",
     "@gitbeaker/node": "^34.6.0",
     "@internal/plugin-healthcheck": "0.1.0",

packages/backend/src/index.ts

@@ -33,6 +33,7 @@ import healthcheck from './plugins/healthcheck';
 import effectionInspector from './plugins/effection-inspector';
 import humanitec from './plugins/humanitec';
 import graphql from './plugins/graphql';
+import idp from './plugins/idp';
 import { PluginEnvironment } from './types';
 import { CatalogClient } from '@backstage/catalog-client';
 
@@ -92,6 +93,7 @@ async function main() {
   const searchEnv = useHotMemoize(module, () => createEnv('search'));
   const appEnv = useHotMemoize(module, () => createEnv('app'));
   const humanitecEnv = useHotMemoize(module, () => createEnv('humanitec'));
+  const idpEnv = useHotMemoize(module, () => createEnv('idp'));
 
   const apiRouter = Router();
   apiRouter.use('/catalog', await catalog(catalogEnv));
@@ -104,6 +106,7 @@ async function main() {
   apiRouter.use('/effection-inspector', await effectionInspector(effectionInspectorEnv));
   apiRouter.use('/humanitec', await humanitec(humanitecEnv));
   apiRouter.use('/graphql', await graphql(graphqlEnv));
+  apiRouter.use('/idp', await idp(idpEnv));
   apiRouter.use(notFoundHandler());
 
   const service = createServiceBuilder(module)

packages/backend/src/plugins/catalog.ts

@@ -3,9 +3,7 @@ import {
 } from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
 import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
-import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github';
 import { Router } from 'express';
-import { Duration } from 'luxon';
 import { PluginEnvironment } from '../types';
 
 export default async function createPlugin(
@@ -16,21 +14,6 @@ export default async function createPlugin(
   // incremental builder receives builder because it'll register
   // incremental entity providers with the builder 
   const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder);
-
-  const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ 
-    host: 'github.com', 
-    searchQuery: "created:>1970-01-01 user:thefrontside", 
-    config: env.config 
-  })
-
-  incrementalBuilder.addIncrementalEntityProvider(
-    githubRepositoryProvider,
-    {
-      burstInterval: Duration.fromObject({ seconds: 3 }),
-      burstLength: Duration.fromObject({ seconds: 3 }),
-      restLength: Duration.fromObject({ day: 1 })
-    }
-  )
 
   builder.addProcessor(new ScaffolderEntitiesProcessor());
 

packages/backend/src/plugins/idp.ts

@@ -0,0 +1,78 @@
+import type { Router } from 'express';
+import { createRouter } from '@frontside/backstage-plugin-platform-backend';
+import { createHumanitecPlatformApi } from '@frontside/backstage-plugin-humanitec-common';
+import { PluginEnvironment } from '../types';
+import { Entity } from '@backstage/catalog-model';
+
+export default async function createPlugin({
+  config,
+  logger,
+  discovery,
+  catalog,
+}: PluginEnvironment): Promise<Router> {
+  return await createRouter({
+    executableName: 'idp',
+    logger,
+    discovery,
+    appURL: `${config.getString('app.baseUrl')}/platform`,
+    catalog,
+    platform: {
+      async getRepositoryUrls(ref) {
+        const entity = await ref.load();
+
+        if (entity) {
+          const slug = getGithubProjectSlug(entity);
+          return {
+            ssh: `git@github.com:${slug}.git`,
+            https: `https://github/${slug}.git`
+          }
+        }
+
+        return null;
+      },
+      async getRepositories() {
+        const { items: entities } = await catalog.getEntities();
+
+        const repositories = entities.flatMap(entity => {
+          const slug = getGithubProjectSlug(entity);
+          if (slug) {
+            return [{
+              componentRef: getComponentRef(entity),
+              slug,
+              description: entity.metadata.description,
+              url: `https://github/${slug}`,
+            }]
+          }
+          return [];
+        });
+        return {
+          hasNextPage: false,
+          hasPreviousPage: false,
+          beginCursor: '',
+          endCursor: '',
+          items: repositories.map(r => ({
+            cursor: '',
+            value: r
+          }))
+        }
+      },
+      ...createHumanitecPlatformApi({
+        token: config.getString('humanitec.token'),
+      }),
+    },
+  });
+}
+
+function getGithubProjectSlug(entity: Entity) {
+  return entity.metadata
+    && entity.metadata.annotations
+    && entity.metadata.annotations["github.com/project-slug"];
+}
+
+function getComponentRef(entity: Entity) {
+  return [
+    entity.kind !== 'Component' ? entity.kind.toLowerCase() : '',
+    entity.metadata.namespace && entity.metadata.namespace !== 'default' ? entity.metadata.namespace : '',
+    entity.metadata.name
+  ].join('')
+}

packages/backend/src/types.ts

@@ -23,5 +23,5 @@ export type PluginEnvironment = {
   tokenManager: TokenManager;
   scheduler: PluginTaskScheduler;
   permissions: PermissionEvaluator;
-  catalog: CatalogClient
+  catalog: CatalogClient;
 };

plugins/humanitec-common/src/clients/humanitec.ts

@@ -60,6 +60,18 @@ export function createHumanitecClient({ orgId, token }: { token: string; orgId:
       const result = await _fetch('GET', `apps/${appId}/envs/${envId}/resources`);
       return ResourcesResponsePayload.parse(result);
     },
+
+    async getEnvironmentLogsSnapshot(appId: string, envId: string) {
+      type Message = {
+        workload_id: string;
+        container_id: string;
+        payload: string;
+        level: string
+      };
+
+      return await _fetch<Message[]>('GET', `apps/${appId}/envs/${envId.toLowerCase()}/logs?limit=100&invert=true`);
+
+    },
     buildUrl(params: URLs) {
       const baseUrl = `https://app.humanitec.io/orgs/${orgId}`;
       switch (params.resource) {
@@ -89,6 +101,8 @@ export function createHumanitecClient({ orgId, token }: { token: string; orgId:
 
       if (r.ok) {
         return await r.json() as R;
+      } else {
+        console.dir({ error: r.statusText });
       }
 
       throw new FetchError(`Fetch ${method} to ${url} failed.`, r);
@@ -98,4 +112,4 @@ export function createHumanitecClient({ orgId, token }: { token: string; orgId:
       retry: async (e: FetchError) => e.status === 403
     });
   }
-}
+}

plugins/humanitec-common/src/constants.ts

@@ -0,0 +1,2 @@
+export const HUMANITEC_ORG_ID_ANNOTATION = "humanitec.com/orgId";
+export const HUMANITEC_APP_ID_ANNOTATION = "humanitec.com/appId";

plugins/humanitec-common/src/index.ts

@@ -4,4 +4,6 @@ export * from './types/environment';
 export * from './types/resources';
 export * from './types/runtime';
 export * from './clients/fetch-app-info';
-export * from './clients/humanitec';
+export * from './clients/humanitec';
+export * from './platform-api';
+export * from './constants';

plugins/humanitec-common/src/platform-api.ts

@@ -0,0 +1,68 @@
+import type { PlatformApi } from '@frontside/backstage-plugin-platform-backend';
+import type { Entity } from '@backstage/catalog-model';
+
+import { HUMANITEC_APP_ID_ANNOTATION, HUMANITEC_ORG_ID_ANNOTATION } from './constants';
+import { createHumanitecClient } from './clients/humanitec';
+
+
+export type HumanitecPlatformAPI = Pick<PlatformApi, 'getLogs' | 'getEnvironments'>
+export function createHumanitecPlatformApi({ token }: { token: string }): HumanitecPlatformAPI {
+
+  return {
+    async *getLogs(ref, envId) {
+      const entity = await ref.load();
+      const { appId, orgId } = getHumanitecMetadata(entity);
+      const client = createHumanitecClient({ token, orgId });
+      const logs = await client.getEnvironmentLogsSnapshot(appId, envId);
+      const bound = Math.floor(logs.length * .5);
+      const send = logs.slice(0, bound);
+      const stream = logs.slice(bound, logs.length);
+      for (const message of send) {
+        yield message.payload;
+      }
+      for (const message of stream) {
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 1500));
+        yield message.payload;
+      }
+    },
+
+    async getEnvironments(ref) {
+      const entity = await ref.load();
+      const { appId, orgId } = getHumanitecMetadata(entity);
+      const client = createHumanitecClient({ token, orgId });
+      const environments = await client.getEnvironments(appId);
+      return {
+        hasNextPage: false,
+        hasPreviousPage: false,
+        beginCursor: '',
+        endCursor: '',
+        items: environments.map(env => ({
+          cursor: '',
+          value: {
+            id: env.id,
+            name: env.name,
+            type: env.type,
+            url: `https://app.humanitec.io/orgs/${orgId}/apps/${appId}/envs/${env.id}`
+          }
+        })),
+      };
+    }
+  }
+}
+
+function getHumanitecMetadata(entity: Entity) {
+  const {
+    [HUMANITEC_ORG_ID_ANNOTATION]: orgId,
+    [HUMANITEC_APP_ID_ANNOTATION]: appId,
+   } = entity.metadata.annotations ?? {};
+
+  if (!orgId) {
+    throw new Error(`${entity.kind}:${entity.metadata.name} is not a humanitec entity`);
+
+  }
+  if (!appId) {
+    throw new Error(`${entity.kind}:${entity.metadata.name} is not a humanitec entity`);
+
+  }
+  return { orgId, appId };
+}

plugins/platform-backend/.eslintrc.js

@@ -0,0 +1,5 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, {
+  rules: {
+    'prefer-const': 'off'
+  }
+});

plugins/platform-backend/README.md

@@ -0,0 +1,14 @@
+# platform-backend
+
+Welcome to the platform-backend backend plugin!
+
+_This plugin was created through the Backstage CLI_
+
+## Getting started
+
+Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn
+start` in the root directory, and then navigating to [/platform-backend](http://localhost:3000/platform-backend).
+
+You can also serve the plugin in isolation by running `yarn start` in the plugin directory.
+This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads.
+It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory.

plugins/platform-backend/cli/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "deno.enable": true,
+  "deno.lint": false,
+  "deno.unstable": true
+}