From 4daa85386b0525a27aa6ae97053b6ac3222c0260 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Wed, 18 Dec 2024 10:42:39 +0100 Subject: fix: revert lookup refactor from #32930 (#33184) --- lib/util/promises.ts | 2 +- lib/workers/repository/process/fetch.spec.ts | 8 +- lib/workers/repository/process/fetch.ts | 129 ++++++++++----------------- 3 files changed, 51 insertions(+), 88 deletions(-) diff --git a/lib/util/promises.ts b/lib/util/promises.ts index 1b614aa036a..736437bc1db 100644 --- a/lib/util/promises.ts +++ b/lib/util/promises.ts @@ -10,7 +10,7 @@ function isExternalHostError(err: any): err is ExternalHostError { return err instanceof ExternalHostError; } -export function handleMultipleErrors(errors: Error[]): never { +function handleMultipleErrors(errors: Error[]): never { const hostError = errors.find(isExternalHostError); if (hostError) { throw hostError; diff --git a/lib/workers/repository/process/fetch.spec.ts b/lib/workers/repository/process/fetch.spec.ts index e68a4ce76f1..0ad23785fe6 100644 --- a/lib/workers/repository/process/fetch.spec.ts +++ b/lib/workers/repository/process/fetch.spec.ts @@ -4,7 +4,6 @@ import { getConfig } from '../../../config/defaults'; import { MavenDatasource } from '../../../modules/datasource/maven'; import type { PackageFile } from '../../../modules/manager/types'; import { ExternalHostError } from '../../../types/errors/external-host-error'; -import { Result } from '../../../util/result'; import { fetchUpdates } from './fetch'; import * as lookup from './lookup'; @@ -59,21 +58,18 @@ describe('workers/repository/process/fetch', () => { depName: 'abcd', packageName: 'abcd', skipReason: 'ignored', - skipStage: 'lookup', updates: [], }, { depName: 'foo', packageName: 'foo', skipReason: 'disabled', - skipStage: 'lookup', updates: [], }, { depName: 'skipped', packageName: 'skipped', skipReason: 'some-reason', - skipStage: 'lookup', updates: [], }, ], @@ -201,7 +197,7 @@ describe('workers/repository/process/fetch', () => { }, ], }; - lookupUpdates.mockResolvedValueOnce(Result.err(new Error('some error'))); + lookupUpdates.mockRejectedValueOnce(new Error('some error')); await expect( fetchUpdates({ ...config, repoIsOnboarded: true }, packageFiles), @@ -218,7 +214,7 @@ describe('workers/repository/process/fetch', () => { }, ], }; - lookupUpdates.mockResolvedValueOnce(Result.err(new Error('some error'))); + lookupUpdates.mockRejectedValueOnce(new Error('some error')); await expect( fetchUpdates({ ...config, repoIsOnboarded: true }, packageFiles), diff --git a/lib/workers/repository/process/fetch.ts b/lib/workers/repository/process/fetch.ts index 36fd712277c..6b3fcb3860b 100644 --- a/lib/workers/repository/process/fetch.ts +++ b/lib/workers/repository/process/fetch.ts @@ -21,13 +21,6 @@ import type { LookupUpdateConfig, UpdateResult } from './lookup/types'; type LookupResult = Result; -interface LookupTaskResult { - packageFile: PackageFile; - result: LookupResult; -} - -type LookupTask = Promise; - async function lookup( packageFileConfig: RenovateConfig & PackageFile, indep: PackageDependency, @@ -113,89 +106,63 @@ async function lookup( }); } -function createLookupTasks( +async function fetchManagerPackagerFileUpdates( config: RenovateConfig, - managerPackageFiles: Record, -): LookupTask[] { - const lookupTasks: LookupTask[] = []; - - for (const [manager, packageFiles] of Object.entries(managerPackageFiles)) { - const managerConfig = getManagerConfig(config, manager); - - for (const packageFile of packageFiles) { - const packageFileConfig = mergeChildConfig(managerConfig, packageFile); - if (packageFile.extractedConstraints) { - packageFileConfig.constraints = { - ...packageFile.extractedConstraints, - ...config.constraints, - }; - } - - for (const dep of packageFile.deps) { - lookupTasks.push( - lookup(packageFileConfig, dep).then((result) => ({ - packageFile, - result, - })), - ); - } - } + managerConfig: RenovateConfig, + pFile: PackageFile, +): Promise { + const { packageFile } = pFile; + const packageFileConfig = mergeChildConfig(managerConfig, pFile); + if (pFile.extractedConstraints) { + packageFileConfig.constraints = { + ...pFile.extractedConstraints, + ...config.constraints, + }; } + const { manager } = packageFileConfig; + const queue = pFile.deps.map( + (dep) => async (): Promise => { + const updates = await lookup(packageFileConfig, dep); + return updates.unwrapOrThrow(); + }, + ); + logger.trace( + { manager, packageFile, queueLength: queue.length }, + 'fetchManagerPackagerFileUpdates starting with concurrency', + ); - return lookupTasks; + pFile.deps = await p.all(queue); + logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished'); } -export async function fetchUpdates( +async function fetchManagerUpdates( config: RenovateConfig, - managerPackageFiles: Record, + packageFiles: Record, + manager: string, ): Promise { - logger.debug( - { baseBranch: config.baseBranch }, - 'Starting package releases lookups', + const managerConfig = getManagerConfig(config, manager); + const queue = packageFiles[manager].map( + (pFile) => (): Promise => + fetchManagerPackagerFileUpdates(config, managerConfig, pFile), ); + logger.trace( + { manager, queueLength: queue.length }, + 'fetchManagerUpdates starting', + ); + await p.all(queue); + logger.trace({ manager }, 'fetchManagerUpdates finished'); +} - const allTasks = createLookupTasks(config, managerPackageFiles); - - const fetchResults = await Promise.all(allTasks); - - const errors: Error[] = []; - - type PackageDeps = WeakMap; - const packageDeps: PackageDeps = new WeakMap(); - - // Separate good results from errors - for (const { packageFile, result } of fetchResults) { - const { val: dep, err } = result.unwrap(); - if (dep) { - let deps = packageDeps.get(packageFile); - if (!deps) { - deps = []; - packageDeps.set(packageFile, deps); - } - if (dep.skipReason && !dep.skipStage) { - dep.skipStage = 'lookup'; - } - deps.push(dep); - } else { - errors.push(err); - } - } - - if (errors.length) { - p.handleMultipleErrors(errors); - } - - // Assign fetched deps back to packageFiles - for (const packageFiles of Object.values(managerPackageFiles)) { - for (const packageFile of packageFiles) { - const packageFileDeps = packageDeps.get(packageFile); - if (packageFileDeps) { - packageFile.deps = packageFileDeps; - } - } - } - - PackageFiles.add(config.baseBranch!, { ...managerPackageFiles }); +export async function fetchUpdates( + config: RenovateConfig, + packageFiles: Record, +): Promise { + const managers = Object.keys(packageFiles); + const allManagerJobs = managers.map((manager) => + fetchManagerUpdates(config, packageFiles, manager), + ); + await Promise.all(allManagerJobs); + PackageFiles.add(config.baseBranch!, { ...packageFiles }); logger.debug( { baseBranch: config.baseBranch }, 'Package releases lookups complete', -- cgit v1.2.3