From 10a4a8bb26caa4753bc0ab5a1f599338a852e31b Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Tue, 7 May 2024 00:01:01 +0545 Subject: [PATCH] feat(self-hosted): `autodiscoverRepoSort` and `autodiscoverRepoOrder` (#28738) Co-authored-by: Rhys Arkins --- docs/usage/self-hosted-configuration.md | 12 +++++++ docs/usage/self-hosted-experimental.md | 37 ------------------- lib/config/global.ts | 2 ++ lib/config/options/index.ts | 20 +++++++++++ lib/config/types.ts | 3 ++ lib/modules/platform/gitea/index.spec.ts | 10 +++--- lib/modules/platform/gitea/index.ts | 40 ++++++++++++++++----- lib/modules/platform/gitea/readme.md | 2 +- lib/modules/platform/gitea/types.ts | 6 +--- lib/modules/platform/types.ts | 11 ++++++ lib/workers/global/autodiscover.ts | 2 ++ lib/workers/global/config/parse/env.spec.ts | 12 +++++++ lib/workers/global/config/parse/env.ts | 25 +++++++++++++ 13 files changed, 124 insertions(+), 58 deletions(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index ffead1ce362a4e..b96039d95cb794 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -233,6 +233,18 @@ This feature is useful for users who want Renovate to only work on repositories The `autodiscoverProjects` config option takes an array of minimatch-compatible globs or RE2-compatible regex strings. For more details on this syntax see Renovate's [string pattern matching documentation](./string-pattern-matching.md). +## autodiscoverRepoOrder + +The order method for autodiscover server side repository search. + +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + +## autodiscoverRepoSort + +The sort method for autodiscover server side repository search. + +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + ## autodiscoverTopics Some platforms allow you to add tags, or topics, to repositories and retrieve repository lists by specifying those diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index 2e59653983c624..f6118f1f15659e 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -32,43 +32,6 @@ Skipping the check will speed things up, but may result in versions being return If set to any value, Renovate will always paginate requests to GitHub fully, instead of stopping after 10 pages. -## `RENOVATE_X_AUTODISCOVER_REPO_ORDER` - - -!!! note - For the Forgejo and Gitea platform only. - -The order method for autodiscover server side repository search. - -> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. - -Allowed values: - -- `asc` -- `desc` - -Default value: `asc`. - -## `RENOVATE_X_AUTODISCOVER_REPO_SORT` - - -!!! note - For the Forgejo and Gitea platform only. - -The sort method for autodiscover server side repository search. - -> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. - -Allowed values: - -- `alpha` -- `created` -- `updated` -- `size` -- `id` - -Default value: `alpha`. - ## `RENOVATE_X_DELETE_CONFIG_FILE` If `true` Renovate tries to delete the self-hosted config file after reading it. diff --git a/lib/config/global.ts b/lib/config/global.ts index c713f8635fd0ba..01bd31a965f646 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -33,6 +33,8 @@ export class GlobalConfig { 'platform', 'endpoint', 'httpCacheTtlDays', + 'autodiscoverRepoSort', + 'autodiscoverRepoOrder', 'userAgent', ]; diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 949db96e066042..08ae23725f5af9 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -22,6 +22,26 @@ const options: RenovateOptions[] = [ globalOnly: true, patternMatch: true, }, + { + name: 'autodiscoverRepoOrder', + description: + 'The order method for autodiscover server side repository search.', + type: 'string', + default: null, + globalOnly: true, + allowedValues: ['asc', 'desc'], + supportedPlatforms: ['gitea'], + }, + { + name: 'autodiscoverRepoSort', + description: + 'The sort method for autodiscover server side repository search.', + type: 'string', + default: null, + globalOnly: true, + allowedValues: ['alpha', 'created', 'updated', 'size', 'id'], + supportedPlatforms: ['gitea'], + }, { name: 'allowedEnv', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index f900fae3b80bc6..2aa2cd155230eb 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -2,6 +2,7 @@ import type { LogLevel } from 'bunyan'; import type { PlatformId } from '../constants'; import type { LogLevelRemap } from '../logger/types'; import type { CustomManager } from '../modules/manager/custom/types'; +import type { RepoSortMethod, SortMethod } from '../modules/platform/types'; import type { HostRule } from '../types'; import type { GitNoVerifyOption } from '../util/git/types'; import type { MergeConfidence } from '../util/merge-confidence/types'; @@ -159,6 +160,8 @@ export interface RepoGlobalConfig { privateKey?: string; privateKeyOld?: string; httpCacheTtlDays?: number; + autodiscoverRepoSort?: RepoSortMethod; + autodiscoverRepoOrder?: SortMethod; userAgent?: string; } diff --git a/lib/modules/platform/gitea/index.spec.ts b/lib/modules/platform/gitea/index.spec.ts index 77b6cb695af810..f74c4270c15606 100644 --- a/lib/modules/platform/gitea/index.spec.ts +++ b/lib/modules/platform/gitea/index.spec.ts @@ -224,9 +224,6 @@ describe('modules/platform/gitea/index', () => { hostRules.clear(); setBaseUrl('https://gitea.renovatebot.com/'); - - delete process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT; - delete process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER; }); async function initFakePlatform( @@ -421,8 +418,6 @@ describe('modules/platform/gitea/index', () => { }); it('Sorts repos', async () => { - process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT = 'updated'; - process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER = 'desc'; const scope = httpMock .scope('https://gitea.com/api/v1') .get('/repos/search') @@ -438,7 +433,10 @@ describe('modules/platform/gitea/index', () => { }); await initFakePlatform(scope); - const repos = await gitea.getRepos(); + const repos = await gitea.getRepos({ + sort: 'updated', + order: 'desc', + }); expect(repos).toEqual(['a/b', 'c/d']); }); }); diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 1117fe91cd54e0..b4859dfe99c140 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -34,6 +34,8 @@ import type { Pr, RepoParams, RepoResult, + RepoSortMethod, + SortMethod, UpdatePrConfig, } from '../types'; import { repoFingerprint } from '../util'; @@ -49,8 +51,6 @@ import type { PRMergeMethod, PRUpdateParams, Repo, - RepoSortMethod, - SortMethod, } from './types'; import { DRAFT_PREFIX, @@ -159,7 +159,17 @@ async function lookupLabelByName(name: string): Promise { return labelList.find((l) => l.name === name)?.id ?? null; } -async function fetchRepositories(topic?: string): Promise { +interface FetchRepositoriesArgs { + topic?: string; + sort?: RepoSortMethod; + order?: SortMethod; +} + +async function fetchRepositories({ + topic, + sort, + order, +}: FetchRepositoriesArgs): Promise { const repos = await helper.searchRepos({ uid: botUserID, archived: false, @@ -167,11 +177,11 @@ async function fetchRepositories(topic?: string): Promise { topic: true, q: topic, }), - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT && { - sort: process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT as RepoSortMethod, + ...(sort && { + sort, }), - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER && { - order: process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER as SortMethod, + ...(order && { + order, }), }); return repos.filter(usableRepo).map((r) => r.full_name); @@ -330,7 +340,16 @@ const platform: Platform = { try { if (config?.topics) { logger.debug({ topics: config.topics }, 'Auto-discovering by topics'); - const repos = await map(config.topics, fetchRepositories); + const fetchRepoArgs: FetchRepositoriesArgs[] = config.topics.map( + (topic) => { + return { + topic, + sort: config.sort, + order: config.order, + }; + }, + ); + const repos = await map(fetchRepoArgs, fetchRepositories); return deduplicateArray(repos.flat()); } else if (config?.namespaces) { logger.debug( @@ -348,7 +367,10 @@ const platform: Platform = { ); return deduplicateArray(repos.flat()); } else { - return await fetchRepositories(); + return await fetchRepositories({ + sort: config?.sort, + order: config?.order, + }); } } catch (err) { logger.error({ err }, 'Gitea getRepos() error'); diff --git a/lib/modules/platform/gitea/readme.md b/lib/modules/platform/gitea/readme.md index 3ce5a31cf167fd..b130a4aa68590c 100644 --- a/lib/modules/platform/gitea/readme.md +++ b/lib/modules/platform/gitea/readme.md @@ -48,5 +48,5 @@ Repositories are ignored when one of the following conditions is met: - Pull requests are disabled for that repository You can change the default server-side sort method and order for autodiscover API. -Set those via [`RENOVATE_X_AUTODISCOVER_REPO_SORT`](../../../self-hosted-experimental.md#renovate_x_autodiscover_repo_sort) and [`RENOVATE_X_AUTODISCOVER_REPO_ORDER`](../../../self-hosted-experimental.md#renovate_x_autodiscover_repo_order). +Set those via [`autodiscoverRepoSort`](../../../self-hosted-configuration.md#autodiscoverRepoSort) and [`autodiscoverRepoOrder`](../../../self-hosted-configuration.md#autodiscoverRepoOrder). Read the [Gitea swagger docs](https://try.gitea.io/api/swagger#/repository/repoSearch) for more details. diff --git a/lib/modules/platform/gitea/types.ts b/lib/modules/platform/gitea/types.ts index 32fa78546950d0..3025a89633ec04 100644 --- a/lib/modules/platform/gitea/types.ts +++ b/lib/modules/platform/gitea/types.ts @@ -1,5 +1,5 @@ import type { LongCommitSha } from '../../../util/git/types'; -import type { Pr } from '../types'; +import type { Pr, RepoSortMethod, SortMethod } from '../types'; export interface PrReviewersParams { reviewers?: string[]; @@ -147,10 +147,6 @@ export interface CombinedCommitStatus { statuses: CommitStatus[]; } -export type RepoSortMethod = 'alpha' | 'created' | 'updated' | 'size' | 'id'; - -export type SortMethod = 'asc' | 'desc'; - export interface RepoSearchParams { uid?: number; archived?: boolean; diff --git a/lib/modules/platform/types.ts b/lib/modules/platform/types.ts index d51e27e1e456be..28f622cd61a33c 100644 --- a/lib/modules/platform/types.ts +++ b/lib/modules/platform/types.ts @@ -201,8 +201,19 @@ export type EnsureCommentRemovalConfig = export type EnsureIssueResult = 'updated' | 'created'; +export type RepoSortMethod = + | 'alpha' + | 'created' + | 'updated' + | 'size' + | 'id' + | null; + +export type SortMethod = 'asc' | 'desc' | null; export interface AutodiscoverConfig { topics?: string[]; + sort?: RepoSortMethod; + order?: SortMethod; includeMirrors?: boolean; namespaces?: string[]; projects?: string[]; diff --git a/lib/workers/global/autodiscover.ts b/lib/workers/global/autodiscover.ts index e6e633df551a9b..e4da1e8c403b5b 100644 --- a/lib/workers/global/autodiscover.ts +++ b/lib/workers/global/autodiscover.ts @@ -38,6 +38,8 @@ export async function autodiscoverRepositories( // Autodiscover list of repositories let discovered = await platform.getRepos({ topics: config.autodiscoverTopics, + sort: config.autodiscoverRepoSort, + order: config.autodiscoverRepoOrder, includeMirrors: config.includeMirrors, namespaces: config.autodiscoverNamespaces, projects: config.autodiscoverProjects, diff --git a/lib/workers/global/config/parse/env.spec.ts b/lib/workers/global/config/parse/env.spec.ts index 664ca142995279..1acaa0b5d5d040 100644 --- a/lib/workers/global/config/parse/env.spec.ts +++ b/lib/workers/global/config/parse/env.spec.ts @@ -267,6 +267,18 @@ describe('workers/global/config/parse/env', () => { expect(config.token).toBe('a'); }); + it('massages converted experimental env vars', async () => { + const envParam: NodeJS.ProcessEnv = { + RENOVATE_X_AUTODISCOVER_REPO_SORT: 'alpha', + RENOVATE_X_DOCKER_MAX_PAGES: '10', + RENOVATE_AUTODISCOVER_REPO_ORDER: 'desc', + }; + const config = await env.getConfig(envParam); + expect(config.autodiscoverRepoSort).toBe('alpha'); + expect(config.autodiscoverRepoOrder).toBe('desc'); + expect(config.dockerMaxPages).toBeUndefined(); + }); + describe('RENOVATE_CONFIG tests', () => { let processExit: jest.SpyInstance; diff --git a/lib/workers/global/config/parse/env.ts b/lib/workers/global/config/parse/env.ts index cc1c4f8fb38405..a37141cff4bba7 100644 --- a/lib/workers/global/config/parse/env.ts +++ b/lib/workers/global/config/parse/env.ts @@ -83,6 +83,30 @@ function massageEnvKeyValues(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { return result; } +const convertedExperimentalEnvVars = [ + 'RENOVATE_X_AUTODISCOVER_REPO_SORT', + 'RENOVATE_X_AUTODISCOVER_REPO_ORDER', +]; + +/** + * Massages the experimental env vars which have been converted to config options + * + * e.g. RENOVATE_X_AUTODISCOVER_REPO_SORT -> RENOVATE_AUTODISCOVER_REPO_SORT + */ +function massageConvertedExperimentalVars( + env: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + const result = { ...env }; + for (const key of convertedExperimentalEnvVars) { + if (env[key] !== undefined) { + const newKey = key.replace('RENOVATE_X_', 'RENOVATE_'); + result[newKey] = env[key]; + delete result[key]; + } + } + return result; +} + export async function getConfig( inputEnv: NodeJS.ProcessEnv, ): Promise { @@ -91,6 +115,7 @@ export async function getConfig( env = renameEnvKeys(env); // massage the values of migrated configuration keys env = massageEnvKeyValues(env); + env = massageConvertedExperimentalVars(env); const options = getOptions();