From 6f31eccfa0b27c31f326c96998c2fc5c58e0e318 Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 11 Sep 2022 20:04:42 -0400 Subject: [PATCH 1/4] Ingesting lots of organizations --- packages/backend/src/plugins/catalog.ts | 7 +- .../incremental-ingestion-github/package.json | 3 + .../providers/repository-entity-provider.ts | 211 ++++++++++++------ yarn.lock | 78 +++++++ 4 files changed, 229 insertions(+), 70 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 61f287cba4..31ade2a9bd 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -18,9 +18,10 @@ export default async function createPlugin( const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ - host: 'github.com', - searchQuery: "created:>1970-01-01 user:thefrontside", - config: env.config + host: 'github.com', + logger: env.logger, + config: env.config, + organizations: ['thefrontside', 'microstates'] }) incrementalBuilder.addIncrementalEntityProvider( diff --git a/plugins/incremental-ingestion-github/package.json b/plugins/incremental-ingestion-github/package.json index a813511123..0927c3f5ee 100644 --- a/plugins/incremental-ingestion-github/package.json +++ b/plugins/incremental-ingestion-github/package.json @@ -32,12 +32,14 @@ "@graphql-codegen/near-operation-file-preset": "^2.4.1", "@graphql-codegen/typescript-operations": "^2.5.3", "@octokit/graphql": "^4.8.0", + "@octokit/rest": "^19.0.4", "@types/express": "*", "assert-ts": "^0.3.4", "express": "^4.17.1", "express-promise-router": "^4.1.0", "graphql": "^16.5.0", "node-fetch": "^2.6.7", + "parse-link-header": "^2.0.0", "slugify": "^1.6.5", "winston": "^3.2.1", "yn": "^4.0.0" @@ -49,6 +51,7 @@ "@graphql-codegen/typescript-document-nodes": "2.3.3", "@octokit/graphql-schema": "^11.1.0", "@types/supertest": "^2.0.8", + "@types/parse-link-header": "^2.0.0", "msw": "^0.42.0", "supertest": "^4.0.2" }, diff --git a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts index 7166243bb7..fe0f42332b 100644 --- a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts +++ b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts @@ -2,53 +2,53 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, DEFAULT_NAMESPACE, str import { Config } from "@backstage/config"; import { DefaultGithubCredentialsProvider, GitHubIntegration, ScmIntegrations } from '@backstage/integration'; import type { EntityIteratorResult, IncrementalEntityProvider } from "@frontside/backstage-plugin-incremental-ingestion-backend"; -import { graphql } from '@octokit/graphql'; +import { Octokit } from '@octokit/rest'; import assert from 'assert-ts'; import slugify from 'slugify'; -import type { RepositorySearchQuery } from "./repository-entity-provider.__generated__"; - -const REPOSITORY_SEARCH_QUERY = /* GraphQL */` - query RepositorySearch($searchQuery: String!, $cursor: String) { - search( - query: $searchQuery - type: REPOSITORY - first: 100 - after: $cursor - ) { - pageInfo { - hasNextPage - endCursor - } - nodes { - ... on Repository { - __typename - id - isArchived - name - nameWithOwner - url - description - visibility - languages(first: 10) { - nodes { - name - } - } - repositoryTopics(first: 10) { - nodes { - topic { +import { RepositoryPrivacy } from "../__generated__/types"; +import type { OrganizationRepositoriesQuery } from "./repository-entity-provider.__generated__"; +import parseLinkHeader from 'parse-link-header'; +import { Logger } from "winston"; + +const ORGANIZATION_REPOSITORIES_QUERY = /* GraphQL */` + query OrganizationRepositories($organization: String!, $privacy: RepositoryPrivacy, $cursor: String) { + organization(login: $organization) { + repositories(first: 100, privacy: $privacy, after: $cursor) { + pageInfo { + hasNextPage + endCursor + } + nodes { + ... on Repository { + __typename + id + isArchived + name + nameWithOwner + url + description + visibility + languages(first: 10) { + nodes { name } } - } - owner { - ... on Organization { - __typename - login + repositoryTopics(first: 10) { + nodes { + topic { + name + } + } } - ... on User { - __typename - login + owner { + ... on Organization { + __typename + login + } + ... on User { + __typename + login + } } } } @@ -64,48 +64,63 @@ const REPOSITORY_SEARCH_QUERY = /* GraphQL */` `; interface GithubRepositoryEntityProviderOptions { - host: string; + logger: Logger; config: Config; - searchQuery: string; + host: string; + organizations?: string[]; + privacy?: RepositoryPrivacy; } interface Context { - client: typeof graphql; + octokit: Octokit; url: string; } interface Cursor { + /** + * Cursor used to paginate repositories + */ cursor: string | null; + /** + * Organiation id used to fetch next organization + */ + since?: number; } -interface GithubRepositoryEntityProviderConstructorOptions { - credentialsProvider: DefaultGithubCredentialsProvider; - host: string; - integration: GitHubIntegration; - searchQuery: string; +interface GithubRepositoryEntityProviderConstructorOptions { + credentialsProvider: DefaultGithubCredentialsProvider; + host: string; + integration: GitHubIntegration; + organizations?: string[]; + privacy?: RepositoryPrivacy; + logger: Logger; } export class GithubRepositoryEntityProvider implements IncrementalEntityProvider { private host: string; private credentialsProvider: DefaultGithubCredentialsProvider; private integration: GitHubIntegration; - private searchQuery: string; + private logger: Logger; + private organizations?: string[]; + private privacy?: RepositoryPrivacy; - static create({ host, config, searchQuery = "created:>1970-01-01" }: GithubRepositoryEntityProviderOptions) { + static create({ host, config, organizations, logger, privacy = RepositoryPrivacy.Public }: GithubRepositoryEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const credentialsProvider = DefaultGithubCredentialsProvider.fromIntegrations(integrations); const integration = integrations.github.byHost(host); assert(integration !== undefined, `Missing Github integration for ${host}`); - return new GithubRepositoryEntityProvider({ credentialsProvider, host, integration, searchQuery }) + return new GithubRepositoryEntityProvider({ credentialsProvider, host, integration, organizations, privacy, logger }); } private constructor(options: GithubRepositoryEntityProviderConstructorOptions) { this.credentialsProvider = options.credentialsProvider; this.host = options.host; this.integration = options.integration; - this.searchQuery = options.searchQuery; + this.organizations = options.organizations; + this.privacy = options.privacy; + this.logger = options.logger; } getProviderName() { @@ -116,30 +131,66 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider const url = `https://${this.host}`; - const { headers } = await this.credentialsProvider.getCredentials({ + const { token } = await this.credentialsProvider.getCredentials({ url, }); - const client = graphql.defaults({ + const octokit = new Octokit({ baseUrl: this.integration.config.apiBaseUrl, - headers, + auth: token, }); - await burst({ client, url }) + await burst({ octokit, url }) } - async next({ client, url }: Context, { cursor }: Cursor = { cursor: null }): Promise> { + async next({ url, octokit }: Context, cursor: Cursor = { cursor: null }): Promise> { - const data = await client(REPOSITORY_SEARCH_QUERY, + const since = cursor.since ?? 0; + + let organization: { login: string, id: number }; + let hasMoreOrgs = false; + + if (this.organizations) { + // array of organizations was passed to entity provider + // treat index in array as id + organization = { + login: this.organizations[since], + id: since + 1 + } + hasMoreOrgs = organization.id < this.organizations.length; + } else { + const response = await octokit.request('GET /organizations', { + since, + per_page: 1 + }); + [organization] = response.data; + const link = parseLinkHeader(response.headers.link); + if (link) { + hasMoreOrgs = !!link.next; + } + } + + this.logger.info(`Current organization`, {...organization, hasMoreOrgs}); + + if (!organization) { + return { + done: true, + cursor: { cursor: null }, + entities: [] + } + } + + const data = await octokit.graphql(ORGANIZATION_REPOSITORIES_QUERY, { - cursor, - searchQuery: this.searchQuery, + organization: organization.login, + cursor: cursor.cursor, + privacy: this.privacy } ); const location = `url:${url}`; - const entities = data.search.nodes?.flatMap(node => node?.__typename === 'Repository' ? [node] : []) + const entities = data.organization?.repositories.nodes?.flatMap(node => node?.__typename === 'Repository' ? [node] : []) .map(node => ({ entity: { apiVersion: 'backstage.io/v1beta1', @@ -167,13 +218,39 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider }, }, locationKey: this.getProviderName(), - })); + })) ?? []; + + if (data.organization?.repositories.pageInfo.hasNextPage) { + const _cursor = { + cursor: data.organization?.repositories.pageInfo.endCursor ?? null, + since: organization.id, + } + this.logger.debug("Has more repositories", _cursor); + return { + done: false, + cursor: _cursor, + entities, + } + } + + if (hasMoreOrgs) { + const _cursor = { + cursor: null, + since: organization.id + }; + this.logger.debug("Has no more repositories but has more orgs", _cursor); + return { + done: false, + cursor: _cursor, + entities + } + } return { - done: !data.search.pageInfo.hasNextPage, - cursor: { cursor: data.search.pageInfo.endCursor ?? null }, - entities: entities ?? [] - }; + done: true, + cursor: { cursor: null }, + entities, + } } } diff --git a/yarn.lock b/yarn.lock index 52af7babde..8b66c658d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5403,6 +5403,13 @@ dependencies: "@octokit/types" "^6.0.3" +"@octokit/auth-token@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-3.0.1.tgz#88bc2baf5d706cb258474e722a720a8365dff2ec" + integrity sha512-/USkK4cioY209wXRpund6HZzHo9GmjakpV9ycOkpMcMxMk7QVcVFVyCMtzvXYiHsB2crgDgrtNYSELYFBXhhaA== + dependencies: + "@octokit/types" "^7.0.0" + "@octokit/auth-unauthenticated@^2.0.0", "@octokit/auth-unauthenticated@^2.0.4": version "2.1.0" resolved "https://registry.yarnpkg.com/@octokit/auth-unauthenticated/-/auth-unauthenticated-2.1.0.tgz#ef97de366836e09f130de4e2205be955f9cf131c" @@ -5424,6 +5431,19 @@ before-after-hook "^2.2.0" universal-user-agent "^6.0.0" +"@octokit/core@^4.0.0": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@octokit/core/-/core-4.0.5.tgz#589e68c0a35d2afdcd41dafceab072c2fbc6ab5f" + integrity sha512-4R3HeHTYVHCfzSAi0C6pbGXV8UDI5Rk+k3G7kLVNckswN9mvpOzW9oENfjfH3nEmzg8y3AmKmzs8Sg6pLCeOCA== + dependencies: + "@octokit/auth-token" "^3.0.0" + "@octokit/graphql" "^5.0.0" + "@octokit/request" "^6.0.0" + "@octokit/request-error" "^3.0.0" + "@octokit/types" "^7.0.0" + before-after-hook "^2.2.0" + universal-user-agent "^6.0.0" + "@octokit/endpoint@^6.0.1": version "6.0.12" resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-6.0.12.tgz#3b4d47a4b0e79b1027fb8d75d4221928b2d05658" @@ -5459,6 +5479,15 @@ "@octokit/types" "^6.0.3" universal-user-agent "^6.0.0" +"@octokit/graphql@^5.0.0": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-5.0.1.tgz#a06982514ad131fb6fbb9da968653b2233fade9b" + integrity sha512-sxmnewSwAixkP1TrLdE6yRG53eEhHhDTYUykUwdV9x8f91WcbhunIHk9x1PZLALdBZKRPUO2HRcm4kezZ79HoA== + dependencies: + "@octokit/request" "^6.0.0" + "@octokit/types" "^7.0.0" + universal-user-agent "^6.0.0" + "@octokit/oauth-app@^3.3.2", "@octokit/oauth-app@^3.5.1": version "3.7.1" resolved "https://registry.yarnpkg.com/@octokit/oauth-app/-/oauth-app-3.7.1.tgz#578657ee289a5176388582204497870992250894" @@ -5513,6 +5542,11 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-12.10.1.tgz#57b5cc6c7b4e55d8642c93d06401fb1af4839899" integrity sha512-P+SukKanjFY0ZhsK6wSVnQmxTP2eVPPE8OPSNuxaMYtgVzwJZgfGdwlYjf4RlRU4vLEw4ts2fsE2icG4nZ5ddQ== +"@octokit/openapi-types@^13.9.1": + version "13.9.1" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-13.9.1.tgz#433d8c8f441ef031f625131360fe47324ae28457" + integrity sha512-98zOxAAR8MDHjXI2xGKgn/qkZLwfcNjHka0baniuEpN1fCv3kDJeh5qc0mBwim5y31eaPaYer9QikzwOkQq3wQ== + "@octokit/plugin-enterprise-rest@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" @@ -5525,6 +5559,13 @@ dependencies: "@octokit/types" "^6.39.0" +"@octokit/plugin-paginate-rest@^4.0.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-4.2.3.tgz#4c49310d9c1f85451027f807ccdeb5939a3af2ce" + integrity sha512-1RXJZ7hnxSANMtxKSVIEByjhYqqlu2GaKmLJJE/OVDya1aI++hdmXP4ORCUlsN2rt4hJzRYbWizBHlGYKz3dhQ== + dependencies: + "@octokit/types" "^7.3.1" + "@octokit/plugin-request-log@^1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" @@ -5538,6 +5579,14 @@ "@octokit/types" "^6.39.0" deprecation "^2.3.1" +"@octokit/plugin-rest-endpoint-methods@^6.0.0": + version "6.5.2" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-6.5.2.tgz#d5f7f91c9e8261cfefea67a3a35de155806fe311" + integrity sha512-zUscUePMC3KEKyTAfuG/dA6hw4Yn7CncVJs2kM9xc4931Iqk3ZiwHfVwTUnxkqQJIVgeBRYUk3rM4hMfgASUxg== + dependencies: + "@octokit/types" "^7.3.1" + deprecation "^2.3.1" + "@octokit/plugin-retry@^3.0.9": version "3.0.9" resolved "https://registry.yarnpkg.com/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz#ae625cca1e42b0253049102acd71c1d5134788fe" @@ -5606,6 +5655,16 @@ "@octokit/plugin-request-log" "^1.0.4" "@octokit/plugin-rest-endpoint-methods" "^5.12.0" +"@octokit/rest@^19.0.4": + version "19.0.4" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-19.0.4.tgz#fd8bed1cefffa486e9ae46a9dc608ce81bcfcbdd" + integrity sha512-LwG668+6lE8zlSYOfwPj4FxWdv/qFXYBpv79TWIQEpBLKA9D/IMcWsF/U9RGpA3YqMVDiTxpgVpEW3zTFfPFTA== + dependencies: + "@octokit/core" "^4.0.0" + "@octokit/plugin-paginate-rest" "^4.0.0" + "@octokit/plugin-request-log" "^1.0.4" + "@octokit/plugin-rest-endpoint-methods" "^6.0.0" + "@octokit/types@^6.0.1", "@octokit/types@^6.0.3", "@octokit/types@^6.10.0", "@octokit/types@^6.12.2", "@octokit/types@^6.16.1", "@octokit/types@^6.27.1", "@octokit/types@^6.34.0", "@octokit/types@^6.35.0", "@octokit/types@^6.39.0", "@octokit/types@^6.8.2": version "6.40.0" resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.40.0.tgz#f2e665196d419e19bb4265603cf904a820505d0e" @@ -5613,6 +5672,13 @@ dependencies: "@octokit/openapi-types" "^12.10.0" +"@octokit/types@^7.0.0", "@octokit/types@^7.3.1": + version "7.3.1" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-7.3.1.tgz#f25aa9ef3566dac48b7a8527059cfdc962013d0a" + integrity sha512-Vefohn8pHGFYWbSc6du0wXMK/Pmy6h0H4lttBw5WqquEuxjdXwyYX07CeZpJDkzSzpdKxBoWRNuDJGTE+FvtqA== + dependencies: + "@octokit/openapi-types" "^13.9.1" + "@octokit/webhooks-methods@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@octokit/webhooks-methods/-/webhooks-methods-2.0.0.tgz#1108b9ea661ca6c81e4a8bfa63a09eb27d5bc2db" @@ -6570,6 +6636,11 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/parse-link-header@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/parse-link-header/-/parse-link-header-2.0.0.tgz#a94d86ac2d13e3cdef4429c977217084e32378e7" + integrity sha512-KbqcQLdRaawDOfXnwqr6nvhe1MV+Uv/Ww+ViSx7Ujgw9X5qCgObLP52B1ZSJqZD8FK1y/4o+bJQTUrZOynegcg== + "@types/passport@^1.0.3": version "1.0.9" resolved "https://registry.yarnpkg.com/@types/passport/-/passport-1.0.9.tgz#b32fa8f7485dace77a9b58e82d0c92908f6e8387" @@ -17565,6 +17636,13 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" +parse-link-header@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/parse-link-header/-/parse-link-header-2.0.0.tgz#949353e284f8aa01f2ac857a98f692b57733f6b7" + integrity sha512-xjU87V0VyHZybn2RrCX5TIFGxTVZE6zqqZWMPlIKiSKuWh/X5WZdt+w1Ki1nXB+8L/KtL+nZ4iq+sfI6MrhhMw== + dependencies: + xtend "~4.0.1" + parse-package-name@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/parse-package-name/-/parse-package-name-0.1.0.tgz#3f44dd838feb4c2be4bf318bae4477d7706bade4" From e52aa9c2e4a369ca69a7843b3f9375fe516a62da Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 12 Sep 2022 11:59:47 -0400 Subject: [PATCH 2/4] Paginating through all organizations --- .../providers/repository-entity-provider.ts | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts index fe0f42332b..9c3c4015b5 100644 --- a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts +++ b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts @@ -85,6 +85,16 @@ interface Cursor { * Organiation id used to fetch next organization */ since?: number; + + /** + * Current organization login + */ + organization?: string; + + /** + * Are there more orgs to process? + */ + hasMoreOrgs: boolean; } interface GithubRepositoryEntityProviderConstructorOptions { @@ -143,34 +153,42 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider await burst({ octokit, url }) } - async next({ url, octokit }: Context, cursor: Cursor = { cursor: null }): Promise> { + async next({ url, octokit }: Context, cursor: Cursor = { cursor: null, hasMoreOrgs: false }): Promise> { const since = cursor.since ?? 0; + let hasMoreOrgs = cursor.hasMoreOrgs ?? false; let organization: { login: string, id: number }; - let hasMoreOrgs = false; - if (this.organizations) { - // array of organizations was passed to entity provider - // treat index in array as id + if (cursor.organization && cursor.cursor !== null) { organization = { - login: this.organizations[since], - id: since + 1 + login: cursor.organization, + id: since } - hasMoreOrgs = organization.id < this.organizations.length; } else { - const response = await octokit.request('GET /organizations', { - since, - per_page: 1 - }); - [organization] = response.data; - const link = parseLinkHeader(response.headers.link); - if (link) { - hasMoreOrgs = !!link.next; + if (this.organizations) { + // array of organizations was passed to entity provider + // treat index in array as id + organization = { + login: this.organizations[since], + id: since + 1 + } + hasMoreOrgs = organization.id < this.organizations.length; + } else { + const response = await octokit.request('GET /organizations', { + since, + per_page: 1 + }); + [organization] = response.data; + const link = parseLinkHeader(response.headers.link); + this.logger.debug(`LINK: ${JSON.stringify(link)}`); + if (link) { + hasMoreOrgs = !!link.next; + } } } - this.logger.info(`Current organization`, {...organization, hasMoreOrgs}); + this.logger.info(`Current organization`, { login: organization.login, id: organization.id, hasMoreOrgs}); if (!organization) { return { @@ -224,6 +242,8 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider const _cursor = { cursor: data.organization?.repositories.pageInfo.endCursor ?? null, since: organization.id, + organization: organization.login, + hasMoreOrgs } this.logger.debug("Has more repositories", _cursor); return { @@ -236,7 +256,8 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider if (hasMoreOrgs) { const _cursor = { cursor: null, - since: organization.id + since: organization.id, + hasMoreOrgs }; this.logger.debug("Has no more repositories but has more orgs", _cursor); return { From ea81f041bd602d126c1fcdafec1da8aedbaecacd Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 9 Oct 2022 10:17:26 -0400 Subject: [PATCH 3/4] Read catalog-info file directly --- packages/backend/src/plugins/catalog.ts | 6 +- .../incremental-ingestion-github/codegen.yml | 3 + .../incremental-ingestion-github/package.json | 1 + .../incremental-ingestion-github/src/index.ts | 2 +- .../providers/discovery-entity-provider.ts | 229 ++++++++++++++ .../providers/repository-entity-provider.ts | 280 ------------------ 6 files changed, 237 insertions(+), 284 deletions(-) create mode 100644 plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts delete mode 100644 plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 31ade2a9bd..77164c085b 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -3,7 +3,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; -import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; +import { GithubDiscoveryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; import { Router } from 'express'; import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; @@ -17,7 +17,7 @@ export default async function createPlugin( // incremental entity providers with the builder const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); - const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ + const githubRepositoryProvider = GithubDiscoveryEntityProvider.create({ host: 'github.com', logger: env.logger, config: env.config, @@ -29,7 +29,7 @@ export default async function createPlugin( { burstInterval: Duration.fromObject({ seconds: 3 }), burstLength: Duration.fromObject({ seconds: 3 }), - restLength: Duration.fromObject({ day: 1 }) + restLength: Duration.fromObject({ minutes: 5 }) } ) diff --git a/plugins/incremental-ingestion-github/codegen.yml b/plugins/incremental-ingestion-github/codegen.yml index 40496eb8d2..757eaa4293 100644 --- a/plugins/incremental-ingestion-github/codegen.yml +++ b/plugins/incremental-ingestion-github/codegen.yml @@ -6,6 +6,9 @@ generates: src/__generated__/types.ts: plugins: - typescript + config: + avoidOptionals: true + declarationKind: 'interface' src/: preset: near-operation-file presetConfig: diff --git a/plugins/incremental-ingestion-github/package.json b/plugins/incremental-ingestion-github/package.json index 0927c3f5ee..9147423396 100644 --- a/plugins/incremental-ingestion-github/package.json +++ b/plugins/incremental-ingestion-github/package.json @@ -25,6 +25,7 @@ "dependencies": { "@backstage/backend-common": "^0.14.0", "@backstage/catalog-model": "^1.0.1", + "@backstage/plugin-catalog-backend": "1.2.0", "@backstage/config": "^1.0.1", "@backstage/integration": "^1.2.1", "@frontside/backstage-plugin-incremental-ingestion-backend": "*", diff --git a/plugins/incremental-ingestion-github/src/index.ts b/plugins/incremental-ingestion-github/src/index.ts index b9d82e5308..44739ea744 100644 --- a/plugins/incremental-ingestion-github/src/index.ts +++ b/plugins/incremental-ingestion-github/src/index.ts @@ -15,4 +15,4 @@ */ export * from './service/router'; -export * from './providers/repository-entity-provider'; +export * from './providers/discovery-entity-provider'; diff --git a/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts new file mode 100644 index 0000000000..92af7d5377 --- /dev/null +++ b/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts @@ -0,0 +1,229 @@ +import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + DefaultGithubCredentialsProvider, + GitHubIntegration, + ScmIntegrations +} from '@backstage/integration'; +import { CatalogProcessorResult, DeferredEntity, parseEntityYaml } from '@backstage/plugin-catalog-backend'; +import type { + EntityIteratorResult, + IncrementalEntityProvider +} from '@frontside/backstage-plugin-incremental-ingestion-backend'; +import { Octokit } from '@octokit/rest'; +import assert from 'assert-ts'; +import { Logger } from 'winston'; +import type { OrganizationRepositoriesQuery } from './discovery-entity-provider.__generated__'; + +interface Context { + octokit: Octokit; + url: string; +} + +interface Cursor { + /** + * Current organization login + */ + login?: string; + + /** + * Cursor used to paginate repositories + */ + endCursor: string | null; +} + +export class GithubDiscoveryEntityProvider + implements IncrementalEntityProvider +{ + private host: string; + private credentialsProvider: DefaultGithubCredentialsProvider; + private integration: GitHubIntegration; + private logger: Logger; + private organizations: string[]; + + static create({ + host, + config, + organizations, + logger, + }: GithubDiscoveryEntityProviderOptions) { + const integrations = ScmIntegrations.fromConfig(config); + const credentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const integration = integrations.github.byHost(host); + + assert(integration !== undefined, `Missing Github integration for ${host}`); + + return new GithubDiscoveryEntityProvider({ + credentialsProvider, + host, + integration, + organizations, + logger, + }); + } + + private constructor( + options: GithubDiscoveryEntityProviderConstructorOptions, + ) { + this.credentialsProvider = options.credentialsProvider; + this.host = options.host; + this.integration = options.integration; + this.organizations = options.organizations; + this.logger = options.logger; + } + + getProviderName() { + return `GithubDiscoveryEntityProvider:${this.host}`; + } + + async around(burst: (context: Context) => Promise) { + const url = `https://${this.host}`; + + const { token } = await this.credentialsProvider.getCredentials({ + url, + }); + + const octokit = new Octokit({ + baseUrl: this.integration.config.apiBaseUrl, + auth: token, + }); + + await burst({ octokit, url }); + } + + async next( + { octokit }: Context, + cursor: Cursor = { endCursor: null }, + ): Promise> { + const login = cursor.login ?? this.organizations[0]; + + this.logger.info("Discovering catalog-info.yaml files", cursor); + + const data = await octokit.graphql( + /* GraphQL */ ` + query OrganizationRepositories( + $login: String! + $endCursor: String + ) { + organization(login: $login) { + repositories(first: 100, after: $endCursor) { + pageInfo { + hasNextPage + endCursor + } + nodes { + url + defaultBranchRef { + name + } + catalogInfo: object(expression: "HEAD:catalog-info.yaml") { + __typename + ... on Blob { + id + text + } + } + } + } + } + rateLimit { + cost + remaining + used + limit + } + } + `, + { + login: login, + endCursor: cursor.endCursor, + }, + ); + + let entities: DeferredEntity[] = []; + + if (data.organization && data.organization.repositories.nodes) { + entities = data.organization.repositories.nodes + ?.flatMap(node => { + const parseResults: CatalogProcessorResult[] = []; + if (node?.catalogInfo?.__typename === 'Blob') { + if (node.catalogInfo.text) { + const location = { type: 'url', target: `${node.url}/blob/${node.defaultBranchRef?.name}/catalog-info.yaml` }; + const content = Buffer.from(node.catalogInfo.text, 'utf8'); + for (const parseResult of parseEntityYaml(content, location)) { + parseResults.push(parseResult); + } + } + } + return parseResults; + }) + // TODO: convert error type into IngestionError + .flatMap(result => result.type === 'entity' ? [{ + entity: { + ...result.entity, + metadata: { + ...result.entity.metadata, + annotations: { + ...result.entity.metadata.annotations, + [ANNOTATION_LOCATION]: `url:${result.location.target}`, + [ANNOTATION_ORIGIN_LOCATION]: this.getProviderName(), + } + } + }, + locationRef: `url:${result.location.target}` + }] : []); + } + + this.logger.info(`Discovered ${entities.length} entities`, cursor); + + if (data.organization?.repositories.pageInfo.hasNextPage) { + const nextPage = { + login, + endCursor: data.organization.repositories.pageInfo.endCursor ?? null, + }; + this.logger.debug(`Organization has more repositories - continue to the next page`, nextPage); + return { + done: false, + cursor: nextPage, + entities + } + } + + const nextOrganization = this.organizations[this.organizations.indexOf(login) + 1] + + if (nextOrganization) { + const nextPage = { + login: nextOrganization, + endCursor: null + }; + this.logger.debug(`Last page for current organization`, nextPage) + return { + done: false, + cursor: nextPage, + entities, + } + } + + return { + done: true, + cursor: { endCursor: null }, + entities, + }; + } +} + +interface GithubDiscoveryEntityProviderConstructorOptions { + credentialsProvider: DefaultGithubCredentialsProvider; + host: string; + integration: GitHubIntegration; + logger: Logger; + organizations: string[]; +} + +interface GithubDiscoveryEntityProviderOptions { + logger: Logger; + config: Config; + host: string; + organizations: string[]; +} diff --git a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts deleted file mode 100644 index 9c3c4015b5..0000000000 --- a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts +++ /dev/null @@ -1,280 +0,0 @@ -import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, DEFAULT_NAMESPACE, stringifyEntityRef } from "@backstage/catalog-model"; -import { Config } from "@backstage/config"; -import { DefaultGithubCredentialsProvider, GitHubIntegration, ScmIntegrations } from '@backstage/integration'; -import type { EntityIteratorResult, IncrementalEntityProvider } from "@frontside/backstage-plugin-incremental-ingestion-backend"; -import { Octokit } from '@octokit/rest'; -import assert from 'assert-ts'; -import slugify from 'slugify'; -import { RepositoryPrivacy } from "../__generated__/types"; -import type { OrganizationRepositoriesQuery } from "./repository-entity-provider.__generated__"; -import parseLinkHeader from 'parse-link-header'; -import { Logger } from "winston"; - -const ORGANIZATION_REPOSITORIES_QUERY = /* GraphQL */` - query OrganizationRepositories($organization: String!, $privacy: RepositoryPrivacy, $cursor: String) { - organization(login: $organization) { - repositories(first: 100, privacy: $privacy, after: $cursor) { - pageInfo { - hasNextPage - endCursor - } - nodes { - ... on Repository { - __typename - id - isArchived - name - nameWithOwner - url - description - visibility - languages(first: 10) { - nodes { - name - } - } - repositoryTopics(first: 10) { - nodes { - topic { - name - } - } - } - owner { - ... on Organization { - __typename - login - } - ... on User { - __typename - login - } - } - } - } - } - } - rateLimit { - cost - remaining - used - limit - } - } -`; - -interface GithubRepositoryEntityProviderOptions { - logger: Logger; - config: Config; - host: string; - organizations?: string[]; - privacy?: RepositoryPrivacy; -} - -interface Context { - octokit: Octokit; - url: string; -} - -interface Cursor { - /** - * Cursor used to paginate repositories - */ - cursor: string | null; - /** - * Organiation id used to fetch next organization - */ - since?: number; - - /** - * Current organization login - */ - organization?: string; - - /** - * Are there more orgs to process? - */ - hasMoreOrgs: boolean; -} - -interface GithubRepositoryEntityProviderConstructorOptions { - credentialsProvider: DefaultGithubCredentialsProvider; - host: string; - integration: GitHubIntegration; - organizations?: string[]; - privacy?: RepositoryPrivacy; - logger: Logger; -} - -export class GithubRepositoryEntityProvider implements IncrementalEntityProvider { - private host: string; - private credentialsProvider: DefaultGithubCredentialsProvider; - private integration: GitHubIntegration; - private logger: Logger; - private organizations?: string[]; - private privacy?: RepositoryPrivacy; - - static create({ host, config, organizations, logger, privacy = RepositoryPrivacy.Public }: GithubRepositoryEntityProviderOptions) { - const integrations = ScmIntegrations.fromConfig(config); - const credentialsProvider = DefaultGithubCredentialsProvider.fromIntegrations(integrations); - const integration = integrations.github.byHost(host); - - assert(integration !== undefined, `Missing Github integration for ${host}`); - - return new GithubRepositoryEntityProvider({ credentialsProvider, host, integration, organizations, privacy, logger }); - } - - private constructor(options: GithubRepositoryEntityProviderConstructorOptions) { - this.credentialsProvider = options.credentialsProvider; - this.host = options.host; - this.integration = options.integration; - this.organizations = options.organizations; - this.privacy = options.privacy; - this.logger = options.logger; - } - - getProviderName() { - return `GithubRepository:${this.host}`; - } - - async around(burst: (context: Context) => Promise) { - - const url = `https://${this.host}`; - - const { token } = await this.credentialsProvider.getCredentials({ - url, - }); - - const octokit = new Octokit({ - baseUrl: this.integration.config.apiBaseUrl, - auth: token, - }); - - await burst({ octokit, url }) - } - - async next({ url, octokit }: Context, cursor: Cursor = { cursor: null, hasMoreOrgs: false }): Promise> { - - const since = cursor.since ?? 0; - let hasMoreOrgs = cursor.hasMoreOrgs ?? false; - - let organization: { login: string, id: number }; - - if (cursor.organization && cursor.cursor !== null) { - organization = { - login: cursor.organization, - id: since - } - } else { - if (this.organizations) { - // array of organizations was passed to entity provider - // treat index in array as id - organization = { - login: this.organizations[since], - id: since + 1 - } - hasMoreOrgs = organization.id < this.organizations.length; - } else { - const response = await octokit.request('GET /organizations', { - since, - per_page: 1 - }); - [organization] = response.data; - const link = parseLinkHeader(response.headers.link); - this.logger.debug(`LINK: ${JSON.stringify(link)}`); - if (link) { - hasMoreOrgs = !!link.next; - } - } - } - - this.logger.info(`Current organization`, { login: organization.login, id: organization.id, hasMoreOrgs}); - - if (!organization) { - return { - done: true, - cursor: { cursor: null }, - entities: [] - } - } - - const data = await octokit.graphql(ORGANIZATION_REPOSITORIES_QUERY, - { - organization: organization.login, - cursor: cursor.cursor, - privacy: this.privacy - } - ); - - const location = `url:${url}`; - - const entities = data.organization?.repositories.nodes?.flatMap(node => node?.__typename === 'Repository' ? [node] : []) - .map(node => ({ - entity: { - apiVersion: 'backstage.io/v1beta1', - kind: 'GithubRepository', - metadata: { - namespace: DEFAULT_NAMESPACE, - name: normalizeEntityName(node.nameWithOwner), - description: node.description ?? '', - annotations: { - [ANNOTATION_LOCATION]: location, - [ANNOTATION_ORIGIN_LOCATION]: location, - }, - }, - spec: { - url: node.url, - owner: stringifyEntityRef({ - kind: `Github${node.owner.__typename}`, - namespace: DEFAULT_NAMESPACE, - name: node.owner.login - }), - nameWithOwner: node.nameWithOwner, - languages: node.languages?.nodes?.flatMap(_node => _node?.__typename === 'Language' ? [_node] : []).map(_node => _node.name) ?? [], - topics: node.repositoryTopics?.nodes?.flatMap(_node => _node?.__typename === 'RepositoryTopic' ? [_node] : []).map(_node => _node.topic.name) ?? [], - visibility: node.visibility, - }, - }, - locationKey: this.getProviderName(), - })) ?? []; - - if (data.organization?.repositories.pageInfo.hasNextPage) { - const _cursor = { - cursor: data.organization?.repositories.pageInfo.endCursor ?? null, - since: organization.id, - organization: organization.login, - hasMoreOrgs - } - this.logger.debug("Has more repositories", _cursor); - return { - done: false, - cursor: _cursor, - entities, - } - } - - if (hasMoreOrgs) { - const _cursor = { - cursor: null, - since: organization.id, - hasMoreOrgs - }; - this.logger.debug("Has no more repositories but has more orgs", _cursor); - return { - done: false, - cursor: _cursor, - entities - } - } - - return { - done: true, - cursor: { cursor: null }, - entities, - } - } -} - -function normalizeEntityName(name: string = '') { - return slugify(name.replace('/', '__').replace('.', '__dot__')) -} \ No newline at end of file From 191de0daee28bfb9af2cd13f5da4f00333ae2e87 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 10 Oct 2022 16:27:48 -0400 Subject: [PATCH 4/4] Discover all repositories --- .../providers/discovery-entity-provider.ts | 251 ++++++++++++------ 1 file changed, 177 insertions(+), 74 deletions(-) diff --git a/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts index 92af7d5377..79a654c655 100644 --- a/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts +++ b/plugins/incremental-ingestion-github/src/providers/discovery-entity-provider.ts @@ -1,23 +1,32 @@ -import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION } from '@backstage/catalog-model'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { DefaultGithubCredentialsProvider, GitHubIntegration, - ScmIntegrations + ScmIntegrations, } from '@backstage/integration'; -import { CatalogProcessorResult, DeferredEntity, parseEntityYaml } from '@backstage/plugin-catalog-backend'; +import { + CatalogProcessorResult, + DeferredEntity, + parseEntityYaml, +} from '@backstage/plugin-catalog-backend'; import type { EntityIteratorResult, - IncrementalEntityProvider + IncrementalEntityProvider, } from '@frontside/backstage-plugin-incremental-ingestion-backend'; import { Octokit } from '@octokit/rest'; import assert from 'assert-ts'; import { Logger } from 'winston'; -import type { OrganizationRepositoriesQuery } from './discovery-entity-provider.__generated__'; +import type { + OrganizationRepositoriesQuery, + RepositoryNodeFragment, +} from './discovery-entity-provider.__generated__'; interface Context { octokit: Octokit; - url: string; } interface Cursor { @@ -32,6 +41,76 @@ interface Cursor { endCursor: string | null; } +type RepositoryMapper = ( + repository: RepositoryNodeFragment, +) => CatalogProcessorResult[]; + +interface GithubDiscoveryEntityProviderConstructorOptions { + credentialsProvider: DefaultGithubCredentialsProvider; + host: string; + integration: GitHubIntegration; + logger: Logger; + organizations: string[]; + repositoryToEntities: RepositoryMapper; +} + +interface GithubDiscoveryEntityProviderOptions { + logger: Logger; + config: Config; + host: string; + organizations: string[]; + repositoryToEntities: RepositoryMapper; +} + +const defaultRepositoryToEntities: RepositoryMapper = node => { + const parseResults: CatalogProcessorResult[] = []; + if (node?.catalogInfo?.__typename === 'Blob' && !!node.catalogInfo.text) { + const location = { + type: 'url', + target: `${node.url}/blob/${node.defaultBranchRef?.name}/catalog-info.yaml`, + }; + const content = Buffer.from(node.catalogInfo.text, 'utf8'); + for (const parseResult of parseEntityYaml(content, location)) { + parseResults.push(parseResult); + } + return parseResults; + } + + if (node.visibility === 'PUBLIC') { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: node.nameWithOwner.replace('/', '__'), + description: node.description ?? '', + tags: [ + ...(node.languages?.nodes?.flatMap(n => n?.__typename === 'Language' ? [n.name] : []) ?? []), + ...(node.repositoryTopics?.nodes?.flatMap(n => n?.__typename === 'RepositoryTopic' ? [n.topic.name] : []) ?? []), + ].map(tag => tag.toLowerCase()), + annotations: { + 'github.com/project-slug': node.nameWithOwner, + } + }, + spec: { + type: 'library', + owner: node.owner.login, + lifecycle: node.isArchived ? 'deprecated' : 'experimental', + }, + }; + parseResults.push({ + type: 'entity', + entity, + location: { + type: 'url', + target: node.url, + }, + }); + console.dir(entity, { depth: 3 }); + } + + return parseResults; +}; + export class GithubDiscoveryEntityProvider implements IncrementalEntityProvider { @@ -40,12 +119,14 @@ export class GithubDiscoveryEntityProvider private integration: GitHubIntegration; private logger: Logger; private organizations: string[]; + private repositoryToEntities: RepositoryMapper; static create({ host, config, organizations, logger, + repositoryToEntities = defaultRepositoryToEntities, }: GithubDiscoveryEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const credentialsProvider = @@ -60,6 +141,7 @@ export class GithubDiscoveryEntityProvider integration, organizations, logger, + repositoryToEntities, }); } @@ -71,6 +153,7 @@ export class GithubDiscoveryEntityProvider this.integration = options.integration; this.organizations = options.organizations; this.logger = options.logger; + this.repositoryToEntities = options.repositoryToEntities; } getProviderName() { @@ -89,7 +172,7 @@ export class GithubDiscoveryEntityProvider auth: token, }); - await burst({ octokit, url }); + await burst({ octokit }); } async next( @@ -98,14 +181,57 @@ export class GithubDiscoveryEntityProvider ): Promise> { const login = cursor.login ?? this.organizations[0]; - this.logger.info("Discovering catalog-info.yaml files", cursor); + this.logger.info('Discovering catalog-info.yaml files', cursor); const data = await octokit.graphql( /* GraphQL */ ` - query OrganizationRepositories( - $login: String! - $endCursor: String - ) { + fragment RepositoryNode on Repository { + url + defaultBranchRef { + name + } + ... on Repository { + __typename + isArchived + name + nameWithOwner + url + description + visibility + languages(first: 10) { + nodes { + __typename + name + } + } + repositoryTopics(first: 10) { + nodes { + __typename + topic { + name + } + } + } + owner { + ... on Organization { + __typename + login + } + ... on User { + __typename + login + } + } + } + catalogInfo: object(expression: "HEAD:catalog-info.yaml") { + __typename + ... on Blob { + id + text + } + } + } + query OrganizationRepositories($login: String!, $endCursor: String) { organization(login: $login) { repositories(first: 100, after: $endCursor) { pageInfo { @@ -113,17 +239,7 @@ export class GithubDiscoveryEntityProvider endCursor } nodes { - url - defaultBranchRef { - name - } - catalogInfo: object(expression: "HEAD:catalog-info.yaml") { - __typename - ... on Blob { - id - text - } - } + ...RepositoryNode } } } @@ -145,35 +261,33 @@ export class GithubDiscoveryEntityProvider if (data.organization && data.organization.repositories.nodes) { entities = data.organization.repositories.nodes - ?.flatMap(node => { - const parseResults: CatalogProcessorResult[] = []; - if (node?.catalogInfo?.__typename === 'Blob') { - if (node.catalogInfo.text) { - const location = { type: 'url', target: `${node.url}/blob/${node.defaultBranchRef?.name}/catalog-info.yaml` }; - const content = Buffer.from(node.catalogInfo.text, 'utf8'); - for (const parseResult of parseEntityYaml(content, location)) { - parseResults.push(parseResult); - } - } - } - return parseResults; - }) - // TODO: convert error type into IngestionError - .flatMap(result => result.type === 'entity' ? [{ - entity: { - ...result.entity, - metadata: { - ...result.entity.metadata, - annotations: { - ...result.entity.metadata.annotations, - [ANNOTATION_LOCATION]: `url:${result.location.target}`, - [ANNOTATION_ORIGIN_LOCATION]: this.getProviderName(), - } - } - }, - locationRef: `url:${result.location.target}` - }] : []); - } + ?.flatMap(node => + node?.__typename === 'Repository' + ? this.repositoryToEntities(node) + : [], + ) + // TODO: convert error type into IngestionError + .flatMap(result => + result.type === 'entity' + ? [ + { + entity: { + ...result.entity, + metadata: { + ...result.entity.metadata, + annotations: { + ...result.entity.metadata.annotations, + [ANNOTATION_LOCATION]: `url:${result.location.target}`, + [ANNOTATION_ORIGIN_LOCATION]: this.getProviderName(), + }, + }, + }, + locationRef: `url:${result.location.target}`, + }, + ] + : [], + ); + } this.logger.info(`Discovered ${entities.length} entities`, cursor); @@ -182,27 +296,31 @@ export class GithubDiscoveryEntityProvider login, endCursor: data.organization.repositories.pageInfo.endCursor ?? null, }; - this.logger.debug(`Organization has more repositories - continue to the next page`, nextPage); + this.logger.debug( + `Organization has more repositories - continue to the next page`, + nextPage, + ); return { done: false, cursor: nextPage, - entities - } + entities, + }; } - const nextOrganization = this.organizations[this.organizations.indexOf(login) + 1] + const nextOrganization = + this.organizations[this.organizations.indexOf(login) + 1]; if (nextOrganization) { const nextPage = { login: nextOrganization, - endCursor: null + endCursor: null, }; - this.logger.debug(`Last page for current organization`, nextPage) + this.logger.debug(`Last page for current organization`, nextPage); return { done: false, cursor: nextPage, entities, - } + }; } return { @@ -212,18 +330,3 @@ export class GithubDiscoveryEntityProvider }; } } - -interface GithubDiscoveryEntityProviderConstructorOptions { - credentialsProvider: DefaultGithubCredentialsProvider; - host: string; - integration: GitHubIntegration; - logger: Logger; - organizations: string[]; -} - -interface GithubDiscoveryEntityProviderOptions { - logger: Logger; - config: Config; - host: string; - organizations: string[]; -}