diff --git a/README.md b/README.md index fa30cdf..c8bbf93 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Archived repositories are filtered out. > :warning: GitHub API are rate limited, and search API in particular has the additional [secondary rate limit](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits). _package-adoption_ implements the [best practices guidelines](https://docs.github.com/en/rest/guides/best-practices-for-integrators#dealing-with-secondary-rate-limits) to deal with it, but you should know that limitations could happen in any case. > :warning: GitHub search API are not 100% reliable and sometimes return deleted / outdated files or multiple versions of the same file. The library version in the output could be inaccurate for this reason. -> There is a [known issue](https://github.com/community/community/discussions/20633#discussioncomment-3735796) with package names with a scope containing hyphen character, e.g. `@typescript-eslint/parser`. +> There is a [known issue](https://github.com/community/community/discussions/20633#discussioncomment-3735796) with package names with a scope containing hyphen character, e.g. `@typescript-eslint/parser`. A slower version of the main scan function will be automatically used to handle this case. ## Install diff --git a/package-lock.json b/package-lock.json index 59fc76b..da40fd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "package-adoption", - "version": "1.3.8", + "version": "1.3.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "package-adoption", - "version": "1.3.8", + "version": "1.3.9", "license": "MIT", "dependencies": { "@octokit/plugin-retry": "^4.0.3", diff --git a/src/getFilteredReposWithPackageForOrg.ts b/src/getFilteredReposWithPackageForOrg.ts index 742a70e..d4c0f9f 100644 --- a/src/getFilteredReposWithPackageForOrg.ts +++ b/src/getFilteredReposWithPackageForOrg.ts @@ -8,6 +8,7 @@ import { readPackageJson } from './readPackageJson'; import { init, octokit } from './octokitInit'; import { isStale } from './isRepoStale'; import { validateConfig } from './validateConfig'; +import { getFilteredReposWithPackageForOrgSlower } from './getFilteredReposWithPackageForOrgSlower'; /** * It takes an organization name and a package name, and returns a list of all the repositories in that @@ -47,7 +48,13 @@ export const getFilteredReposWithPackageForOrg = async ( for await (const { data: items } of foundPackageJsonFiles) { if (items.length === 0) { - console.log(`[package-adoption]: No results for ${pkgName}`); + // workaround to address github search issue https://github.com/community/community/discussions/20633#discussioncomment-3735796 + const repositoriesWithPackage = + await getFilteredReposWithPackageForOrgSlower({ ...config }, octokit); + if (repositoriesWithPackage?.length === 0) { + console.log(`[package-adoption]: No results for ${pkgName}`); + } + return repositoriesWithPackage; } for (let i = 0; i < items.length; i++) { diff --git a/src/getFilteredReposWithPackageForOrgSlower.ts b/src/getFilteredReposWithPackageForOrgSlower.ts new file mode 100644 index 0000000..959b64f --- /dev/null +++ b/src/getFilteredReposWithPackageForOrgSlower.ts @@ -0,0 +1,105 @@ +import path from 'path'; +import { GetResponseDataTypeFromEndpointMethod } from '@octokit/types'; +import { ErrorWithResponse, InputParameters, RelevantRepo } from './types'; +import { readPackageJson } from './readPackageJson'; +import { isStale } from './isRepoStale'; +import { Octokit } from '@octokit/rest'; + +/** + * It takes an organization name and a package name, and returns a list of all the repositories in that + * organization that have that package in their package.json file + * @param {InputParameters} config - InputParameters + * @returns An array of objects with the installation info for the library + */ +export const getFilteredReposWithPackageForOrgSlower = async ( + config: InputParameters, + octokit: Octokit +): Promise => { + const { org, daysUntilStale = 365, pkgName } = config; + const repositoriesWithPackage: RelevantRepo[] = []; + + type IteratorResponseItemDataType = GetResponseDataTypeFromEndpointMethod< + typeof octokit.repos.get + >; + + try { + /* The plain listForOrg API just returns 30 items (a page), we need paginate iterator to get the whole list */ + const allRepos = octokit.paginate.iterator( + 'GET /orgs/:org/repos', + { + org, + type: 'all', + } + ); + + for await (const { data: repos } of allRepos) { + for (let i = 0; i < repos.length; i++) { + const repo = repos[i]; + + if ( + repo.archived === false && + (repo.language === 'TypeScript' || repo.language === 'JavaScript') && + !isStale(repo.pushed_at, daysUntilStale) + ) { + try { + const packageJsonResponse = await octokit.search.code({ + q: `repo:${org}/${repo.name}+filename:package.json`, + }); + const foundFiles = packageJsonResponse.data.items; + + for (let i = 0; i < foundFiles.length; i++) { + const packageJsonFile = foundFiles[i]; + + const pathDirParts = packageJsonFile.path.split('/').slice(0, -1); + + const installationPath = + pathDirParts?.length > 0 ? path.join(...pathDirParts) : 'root'; + + if ( + // The search matches package-lock too + packageJsonFile.name === 'package.json' && + // sometimes GitHub search returns multiple versions of the same file + !repositoriesWithPackage.find( + (relevantRepo) => + relevantRepo.name === repo.name && + relevantRepo.installationPath === installationPath + ) + ) { + const packageJsonData = await readPackageJson({ + org, + repoName: repo.name, + pkgName, + packageJsonPath: packageJsonFile.path, + installationPath, + }); + if (packageJsonData) { + repositoriesWithPackage.push(packageJsonData); + } + } + } + } catch (error) { + const typeSafeError = error as ErrorWithResponse; + + if (typeSafeError.response === null) { + console.error(error); + } else if (typeSafeError.response.status === 404) { + console.error( + `[package-adoption]: No package.json found.\n${typeSafeError?.response?.url}\n\n` + ); + } else if (typeSafeError.response.status === 403) { + // API rate limit exceeded for user ID + console.error( + `[package-adoption]: ${typeSafeError?.response?.data.message}\n${typeSafeError?.response?.url}\n\n` + ); + } else { + console.error(typeSafeError.response); + } + } + } + } + } + return repositoriesWithPackage; + } catch (error) { + console.error('[package-adoption]:', error); + } +};