import URL from 'node:url'; import { setTimeout } from 'timers/promises'; import is from '@sindresorhus/is'; import fs from 'fs-extra'; import semver from 'semver'; import type { Options, SimpleGit, TaskOptions } from 'simple-git'; import { ResetMode, simpleGit } from 'simple-git'; import upath from 'upath'; import { configFileNames } from '../../config/app-strings'; import { GlobalConfig } from '../../config/global'; import type { RenovateConfig } from '../../config/types'; import { CONFIG_VALIDATION, INVALID_PATH, REPOSITORY_CHANGED, REPOSITORY_DISABLED, REPOSITORY_EMPTY, SYSTEM_INSUFFICIENT_DISK_SPACE, TEMPORARY_ERROR, } from '../../constants/error-messages'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import type { GitProtocol } from '../../types/git'; import { incLimitedValue } from '../../workers/global/limits'; import { getCache } from '../cache/repository'; import { newlineRegex, regEx } from '../regex'; import { parseGitAuthor } from './author'; import { getCachedBehindBaseResult, setCachedBehindBaseResult, } from './behind-base-branch-cache'; import { getNoVerify, simpleGitConfig } from './config'; import { getCachedConflictResult, setCachedConflictResult, } from './conflicts-cache'; import { bulkChangesDisallowed, checkForPlatformFailure, handleCommitError, } from './error'; import { getCachedModifiedResult, setCachedModifiedResult, } from './modified-cache'; import { configSigningKey, writePrivateKey } from './private-key'; import type { CommitFilesConfig, CommitResult, LocalConfig, LongCommitSha, PushFilesConfig, StatusResult, StorageConfig, TreeItem, } from './types'; export { setNoVerify } from './config'; export { setPrivateKey } from './private-key'; // Retry parameters const retryCount = 5; const delaySeconds = 3; const delayFactor = 2; // A generic wrapper for simpleGit.* calls to make them more fault-tolerant export async function gitRetry(gitFunc: () => Promise): Promise { let round = 0; let lastError: Error | undefined; while (round <= retryCount) { if (round > 0) { logger.debug(`gitRetry round ${round}`); } try { const res = await gitFunc(); if (round > 1) { logger.debug('Successful retry of git function'); } return res; } catch (err) { lastError = err; logger.debug({ err }, `Git function thrown`); // Try to transform the Error to ExternalHostError const errChecked = checkForPlatformFailure(err); if (errChecked instanceof ExternalHostError) { logger.debug( { err: errChecked }, `ExternalHostError thrown in round ${ round + 1 } of ${retryCount} - retrying in the next round`, ); } else { throw err; } } const nextDelay = delayFactor ^ ((round - 1) * delaySeconds); logger.trace({ nextDelay }, `Delay next round`); await setTimeout(1000 * nextDelay); round++; } // Can't be `undefined` here. // eslint-disable-next-line @typescript-eslint/only-throw-error throw lastError; } async function isDirectory(dir: string): Promise { try { return (await fs.stat(dir)).isDirectory(); } catch { return false; } } async function getDefaultBranch(git: SimpleGit): Promise { // see https://stackoverflow.com/a/62352647/3005034 try { let res = await git.raw(['rev-parse', '--abbrev-ref', 'origin/HEAD']); // istanbul ignore if if (!res) { logger.debug('Could not determine default branch using git rev-parse'); const headPrefix = 'HEAD branch: '; res = (await git.raw(['remote', 'show', 'origin'])) .split('\n') .map((line) => line.trim()) .find((line) => line.startsWith(headPrefix))! .replace(headPrefix, ''); } return res.replace('origin/', '').trim(); } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } if ( err.message.startsWith( 'fatal: ref refs/remotes/origin/HEAD is not a symbolic ref', ) ) { throw new Error(REPOSITORY_EMPTY); } // istanbul ignore if if (err.message.includes("fatal: ambiguous argument 'origin/HEAD'")) { logger.warn({ err }, 'Error getting default branch'); throw new Error(TEMPORARY_ERROR); } throw err; } } let config: LocalConfig = {} as any; // TODO: can be undefined let git: SimpleGit; let gitInitialized: boolean; let submodulesInitizialized: boolean; let privateKeySet = false; export const GIT_MINIMUM_VERSION = '2.33.0'; // git show-current export async function validateGitVersion(): Promise { let version: string | undefined; const globalGit = simpleGit(); try { const { major, minor, patch, installed } = await globalGit.version(); // istanbul ignore if if (!installed) { logger.error('Git not installed'); return false; } version = `${major}.${minor}.${patch}`; } catch (err) /* istanbul ignore next */ { logger.error({ err }, 'Error fetching git version'); return false; } // istanbul ignore if if (!(version && semver.gte(version, GIT_MINIMUM_VERSION))) { logger.error( { detectedVersion: version, minimumVersion: GIT_MINIMUM_VERSION }, 'Git version needs upgrading', ); return false; } logger.debug(`Found valid git version: ${version}`); return true; } async function fetchBranchCommits(): Promise { config.branchCommits = {}; const opts = ['ls-remote', '--heads', config.url]; if (config.extraCloneOpts) { Object.entries(config.extraCloneOpts).forEach((e) => // TODO: types (#22198) opts.unshift(e[0], `${e[1]!}`), ); } try { const lsRemoteRes = await gitRetry(() => git.raw(opts)); logger.trace({ lsRemoteRes }, 'git ls-remote result'); lsRemoteRes .split(newlineRegex) .filter(Boolean) .map((line) => line.trim().split(regEx(/\s+/))) .forEach(([sha, ref]) => { config.branchCommits[ref.replace('refs/heads/', '')] = sha as LongCommitSha; }); logger.trace({ branchCommits: config.branchCommits }, 'branch commits'); } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } logger.debug({ err }, 'git error'); if (err.message?.includes('Please ask the owner to check their account')) { throw new Error(REPOSITORY_DISABLED); } throw err; } } export async function fetchRevSpec(revSpec: string): Promise { await gitRetry(() => git.fetch(['origin', revSpec])); } export async function initRepo(args: StorageConfig): Promise { config = { ...args } as any; config.ignoredAuthors = []; config.additionalBranches = []; config.branchIsModified = {}; git = simpleGit(GlobalConfig.get('localDir'), simpleGitConfig()).env({ ...process.env, LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8', }); gitInitialized = false; submodulesInitizialized = false; await fetchBranchCommits(); } async function resetToBranch(branchName: string): Promise { logger.debug(`resetToBranch(${branchName})`); await git.raw(['reset', '--hard']); await gitRetry(() => git.checkout(branchName)); await git.raw(['reset', '--hard', 'origin/' + branchName]); await git.raw(['clean', '-fd']); } // istanbul ignore next export async function resetToCommit(commit: LongCommitSha): Promise { logger.debug(`resetToCommit(${commit})`); await git.raw(['reset', '--hard', commit]); } async function deleteLocalBranch(branchName: string): Promise { await git.branch(['-D', branchName]); } async function cleanLocalBranches(): Promise { const existingBranches = (await git.raw(['branch'])) .split(newlineRegex) .map((branch) => branch.trim()) .filter((branch) => branch.length) .filter((branch) => !branch.startsWith('* ')); logger.debug({ existingBranches }); for (const branchName of existingBranches) { await deleteLocalBranch(branchName); } } export function setGitAuthor(gitAuthor: string | undefined): void { const gitAuthorParsed = parseGitAuthor( gitAuthor ?? 'Renovate Bot ', ); if (!gitAuthorParsed) { const error = new Error(CONFIG_VALIDATION); error.validationSource = 'None'; error.validationError = 'Invalid gitAuthor'; error.validationMessage = `\`gitAuthor\` is not parsed as valid RFC5322 format: \`${gitAuthor!}\``; throw error; } config.gitAuthorName = gitAuthorParsed.name; config.gitAuthorEmail = gitAuthorParsed.address; } export async function writeGitAuthor(): Promise { const { gitAuthorName, gitAuthorEmail, writeGitDone } = config; // istanbul ignore if if (writeGitDone) { return; } config.writeGitDone = true; try { if (gitAuthorName) { logger.debug(`Setting git author name: ${gitAuthorName}`); await git.addConfig('user.name', gitAuthorName); } if (gitAuthorEmail) { logger.debug(`Setting git author email: ${gitAuthorEmail}`); await git.addConfig('user.email', gitAuthorEmail); } } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } logger.debug( { err, gitAuthorName, gitAuthorEmail }, 'Error setting git author config', ); throw new Error(TEMPORARY_ERROR); } } export function setUserRepoConfig({ gitIgnoredAuthors, gitAuthor, }: RenovateConfig): void { config.ignoredAuthors = gitIgnoredAuthors ?? []; setGitAuthor(gitAuthor); } export async function getSubmodules(): Promise { try { return ( (await git.raw([ 'config', '--file', '.gitmodules', '--get-regexp', '\\.path', ])) || '' ) .trim() .split(regEx(/[\n\s]/)) .filter((_e: string, i: number) => i % 2); } catch (err) /* istanbul ignore next */ { logger.warn({ err }, 'Error getting submodules'); return []; } } export async function cloneSubmodules(shouldClone: boolean): Promise { if (!shouldClone || submodulesInitizialized) { return; } submodulesInitizialized = true; await syncGit(); const submodules = await getSubmodules(); for (const submodule of submodules) { try { logger.debug(`Cloning git submodule at ${submodule}`); await gitRetry(() => git.submoduleUpdate(['--init', '--recursive', submodule]), ); } catch (err) { logger.warn( { err }, `Unable to initialise git submodule at ${submodule}`, ); } } } export function isCloned(): boolean { return gitInitialized; } export async function syncGit(): Promise { if (gitInitialized) { // istanbul ignore if if (process.env.RENOVATE_X_CLEAR_HOOKS) { await git.raw(['config', 'core.hooksPath', '/dev/null']); } return; } // istanbul ignore if: failsafe if (GlobalConfig.get('platform') === 'local') { throw new Error('Cannot sync git when platform=local'); } gitInitialized = true; const localDir = GlobalConfig.get('localDir')!; logger.debug(`Initializing git repository into ${localDir}`); const gitHead = upath.join(localDir, '.git/HEAD'); let clone = true; if (await fs.pathExists(gitHead)) { try { await git.raw(['remote', 'set-url', 'origin', config.url]); await resetToBranch(await getDefaultBranch(git)); const fetchStart = Date.now(); await gitRetry(() => git.pull()); await gitRetry(() => git.fetch()); config.currentBranch = config.currentBranch || (await getDefaultBranch(git)); await resetToBranch(config.currentBranch); await cleanLocalBranches(); await gitRetry(() => git.raw(['remote', 'prune', 'origin'])); const durationMs = Math.round(Date.now() - fetchStart); logger.info({ durationMs }, 'git fetch completed'); clone = false; } catch (err) /* istanbul ignore next */ { if (err.message === REPOSITORY_EMPTY) { throw err; } logger.info({ err }, 'git fetch error'); } } if (clone) { const cloneStart = Date.now(); try { const opts: string[] = []; if (config.defaultBranch) { opts.push('-b', config.defaultBranch); } if (config.fullClone) { logger.debug('Performing full clone'); } else { logger.debug('Performing blobless clone'); opts.push('--filter=blob:none'); } if (config.extraCloneOpts) { Object.entries(config.extraCloneOpts).forEach((e) => // TODO: types (#22198) opts.push(e[0], `${e[1]!}`), ); } const emptyDirAndClone = async (): Promise => { await fs.emptyDir(localDir); await git.clone(config.url, '.', opts); }; await gitRetry(() => emptyDirAndClone()); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'git clone error'); if (err.message?.includes('No space left on device')) { throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE); } if (err.message === REPOSITORY_EMPTY) { throw err; } throw new ExternalHostError(err, 'git'); } const durationMs = Math.round(Date.now() - cloneStart); logger.debug({ durationMs }, 'git clone completed'); } try { config.currentBranchSha = ( await git.raw(['rev-parse', 'HEAD']) ).trim() as LongCommitSha; } catch (err) /* istanbul ignore next */ { if (err.message?.includes('fatal: not a git repository')) { throw new Error(REPOSITORY_CHANGED); } throw err; } // This will only happen now if set in global config await cloneSubmodules(!!config.cloneSubmodules); try { const latestCommit = (await git.log({ n: 1 })).latest; logger.debug({ latestCommit }, 'latest repository commit'); } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } if (err.message.includes('does not have any commits yet')) { throw new Error(REPOSITORY_EMPTY); } logger.warn({ err }, 'Cannot retrieve latest commit'); } config.currentBranch = config.currentBranch ?? config.defaultBranch ?? (await getDefaultBranch(git)); delete getCache()?.semanticCommits; } // istanbul ignore next export async function getRepoStatus(path?: string): Promise { if (is.string(path)) { const localDir = GlobalConfig.get('localDir'); const localPath = upath.resolve(localDir, path); if (!localPath.startsWith(upath.resolve(localDir))) { logger.warn( { localPath, localDir }, 'Preventing access to file outside the local directory', ); throw new Error(INVALID_PATH); } } await syncGit(); return git.status(path ? [path] : []); } export function branchExists(branchName: string): boolean { return !!config.branchCommits[branchName]; } // Return the commit SHA for a branch export function getBranchCommit(branchName: string): LongCommitSha | null { return config.branchCommits[branchName] || null; } export async function getCommitMessages(): Promise { logger.debug('getCommitMessages'); if (GlobalConfig.get('platform') !== 'local') { await syncGit(); } try { const res = await git.log({ n: 20, format: { message: '%s' }, }); return res.all.map((commit) => commit.message); } catch /* istanbul ignore next */ { return []; } } export async function checkoutBranch( branchName: string, ): Promise { logger.debug(`Setting current branch to ${branchName}`); await syncGit(); try { await gitRetry(() => git.checkout( submodulesInitizialized ? ['-f', '--recurse-submodules', branchName, '--'] : ['-f', branchName, '--'], ), ); config.currentBranch = branchName; config.currentBranchSha = ( await git.raw(['rev-parse', 'HEAD']) ).trim() as LongCommitSha; const latestCommitDate = (await git.log({ n: 1 }))?.latest?.date; if (latestCommitDate) { logger.debug({ branchName, latestCommitDate }, 'latest commit'); } await git.reset(ResetMode.HARD); return config.currentBranchSha; } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } if (err.message?.includes('fatal: ambiguous argument')) { logger.warn({ err }, 'Failed to checkout branch'); throw new Error(TEMPORARY_ERROR); } throw err; } } export async function getFileList(): Promise { await syncGit(); const branch = config.currentBranch; let files: string; try { files = await git.raw(['ls-tree', '-r', branch]); } catch (err) /* istanbul ignore next */ { if (err.message?.includes('fatal: Not a valid object name')) { logger.debug( { err }, 'Branch not found when checking branch list - aborting', ); throw new Error(REPOSITORY_CHANGED); } throw err; } // istanbul ignore if if (!files) { return []; } // submodules are starting with `160000 commit` return files .split(newlineRegex) .filter(is.string) .filter((line) => line.startsWith('100')) .map((line) => line.split(regEx(/\t/)).pop()!); } export function getBranchList(): string[] { return Object.keys(config.branchCommits ?? /* istanbul ignore next */ {}); } export async function isBranchBehindBase( branchName: string, baseBranch: string, ): Promise { const baseBranchSha = getBranchCommit(baseBranch); const branchSha = getBranchCommit(branchName); let isBehind = getCachedBehindBaseResult( branchName, branchSha, baseBranch, baseBranchSha, ); if (isBehind !== null) { logger.debug(`branch.isBehindBase(): using cached result "${isBehind}"`); return isBehind; } logger.debug('branch.isBehindBase(): using git to calculate'); await syncGit(); try { const behindCount = ( await git.raw(['rev-list', '--count', `${branchSha!}..${baseBranchSha!}`]) ).trim(); isBehind = behindCount !== '0'; logger.debug( { baseBranch, branchName }, `branch.isBehindBase(): ${isBehind}`, ); setCachedBehindBaseResult(branchName, isBehind); return isBehind; } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } throw err; } } export async function isBranchModified( branchName: string, baseBranch: string, ): Promise { if (!branchExists(branchName)) { logger.debug('branch.isModified(): no cache'); return false; } // First check local config if (config.branchIsModified[branchName] !== undefined) { return config.branchIsModified[branchName]; } // Second check repository cache const isModified = getCachedModifiedResult( branchName, getBranchCommit(branchName), // branch sha ); if (isModified !== null) { logger.debug(`branch.isModified(): using cached result "${isModified}"`); config.branchIsModified[branchName] = isModified; return isModified; } logger.debug('branch.isModified(): using git to calculate'); await syncGit(); const committedAuthors = new Set(); try { const commits = await git.log([ `origin/${baseBranch}..origin/${branchName}`, ]); for (const commit of commits.all) { committedAuthors.add(commit.author_email); } } catch (err) /* istanbul ignore next */ { if (err.message?.includes('fatal: bad revision')) { logger.debug( { err }, 'Remote branch not found when checking last commit author - aborting run', ); throw new Error(REPOSITORY_CHANGED); } logger.warn({ err }, 'Error checking last author for isBranchModified'); } const { gitAuthorEmail, ignoredAuthors } = config; const includedAuthors = new Set(committedAuthors); if (gitAuthorEmail) { includedAuthors.delete(gitAuthorEmail); } for (const ignoredAuthor of ignoredAuthors) { includedAuthors.delete(ignoredAuthor); } if (includedAuthors.size === 0) { // authors all match - branch has not been modified logger.trace( { branchName, baseBranch, committedAuthors: [...committedAuthors], includedAuthors: [...includedAuthors], gitAuthorEmail, ignoredAuthors, }, 'branch.isModified() = false', ); logger.debug('branch.isModified() = false'); config.branchIsModified[branchName] = false; setCachedModifiedResult(branchName, false); return false; } logger.trace( { branchName, baseBranch, committedAuthors: [...committedAuthors], includedAuthors: [...includedAuthors], gitAuthorEmail, ignoredAuthors, }, 'branch.isModified() = true', ); logger.debug( { branchName, unrecognizedAuthors: [...includedAuthors] }, 'branch.isModified() = true', ); config.branchIsModified[branchName] = true; setCachedModifiedResult(branchName, true); return true; } export async function isBranchConflicted( baseBranch: string, branch: string, ): Promise { logger.debug(`isBranchConflicted(${baseBranch}, ${branch})`); const baseBranchSha = getBranchCommit(baseBranch); const branchSha = getBranchCommit(branch); if (!baseBranchSha || !branchSha) { logger.warn( { baseBranch, branch }, 'isBranchConflicted: branch does not exist', ); return true; } const isConflicted = getCachedConflictResult( branch, branchSha, baseBranch, baseBranchSha, ); if (is.boolean(isConflicted)) { logger.debug( `branch.isConflicted(): using cached result "${isConflicted}"`, ); return isConflicted; } logger.debug('branch.isConflicted(): using git to calculate'); let result = false; await syncGit(); await writeGitAuthor(); const origBranch = config.currentBranch; try { await git.reset(ResetMode.HARD); //TODO: see #18600 if (origBranch !== baseBranch) { await git.checkout(baseBranch); } await git.merge(['--no-commit', '--no-ff', `origin/${branch}`]); } catch (err) { result = true; // istanbul ignore if: not easily testable if (!err?.git?.conflicts?.length) { logger.debug( { baseBranch, branch, err }, 'isBranchConflicted: unknown error', ); } } finally { try { await git.merge(['--abort']); if (origBranch !== baseBranch) { await git.checkout(origBranch); } } catch (err) /* istanbul ignore next */ { logger.debug( { baseBranch, branch, err }, 'isBranchConflicted: cleanup error', ); } } setCachedConflictResult(branch, result); logger.debug(`branch.isConflicted(): ${result}`); return result; } export async function deleteBranch(branchName: string): Promise { await syncGit(); try { const deleteCommand = ['push', '--delete', 'origin', branchName]; if (getNoVerify().includes('push')) { deleteCommand.push('--no-verify'); } await gitRetry(() => git.raw(deleteCommand)); logger.debug(`Deleted remote branch: ${branchName}`); } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } logger.debug(`No remote branch to delete with name: ${branchName}`); } try { await deleteLocalBranch(branchName); // istanbul ignore next logger.debug(`Deleted local branch: ${branchName}`); } catch (err) { const errChecked = checkForPlatformFailure(err); // istanbul ignore if if (errChecked) { throw errChecked; } logger.debug(`No local branch to delete with name: ${branchName}`); } delete config.branchCommits[branchName]; } export async function mergeToLocal(refSpecToMerge: string): Promise { let status: StatusResult | undefined; try { await syncGit(); await writeGitAuthor(); await git.reset(ResetMode.HARD); await gitRetry(() => git.checkout([ '-B', config.currentBranch, 'origin/' + config.currentBranch, ]), ); status = await git.status(); await fetchRevSpec(refSpecToMerge); await gitRetry(() => git.merge(['FETCH_HEAD'])); } catch (err) { logger.debug( { baseBranch: config.currentBranch, baseSha: config.currentBranchSha, refSpecToMerge, status, err, }, 'mergeLocally error', ); throw err; } } export async function mergeBranch(branchName: string): Promise { let status: StatusResult | undefined; try { await syncGit(); await writeGitAuthor(); await git.reset(ResetMode.HARD); await gitRetry(() => git.checkout(['-B', branchName, 'origin/' + branchName]), ); await gitRetry(() => git.checkout([ '-B', config.currentBranch, 'origin/' + config.currentBranch, ]), ); status = await git.status(); await gitRetry(() => git.merge(['--ff-only', branchName])); await gitRetry(() => git.push('origin', config.currentBranch)); incLimitedValue('Commits'); } catch (err) { logger.debug( { baseBranch: config.currentBranch, baseSha: config.currentBranchSha, branchName, branchSha: getBranchCommit(branchName), status, err, }, 'mergeBranch error', ); throw err; } } export async function getBranchLastCommitTime( branchName: string, ): Promise { await syncGit(); try { const time = await git.show(['-s', '--format=%ai', 'origin/' + branchName]); return new Date(Date.parse(time)); } catch (err) { const errChecked = checkForPlatformFailure(err); // istanbul ignore if if (errChecked) { throw errChecked; } return new Date(); } } export async function getBranchFiles( branchName: string, ): Promise { await syncGit(); try { const diff = await gitRetry(() => git.diffSummary([`origin/${branchName}`, `origin/${branchName}^`]), ); return diff.files.map((file) => file.file); } catch (err) /* istanbul ignore next */ { logger.warn({ err }, 'getBranchFiles error'); const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } return null; } } export async function getFile( filePath: string, branchName?: string, ): Promise { await syncGit(); try { const content = await git.show([ 'origin/' + (branchName ?? config.currentBranch) + ':' + filePath, ]); return content; } catch (err) { const errChecked = checkForPlatformFailure(err); // istanbul ignore if if (errChecked) { throw errChecked; } return null; } } export async function getFiles( fileNames: string[], ): Promise> { const fileContentMap: Record = {}; for (const fileName of fileNames) { fileContentMap[fileName] = await getFile(fileName); } return fileContentMap; } export async function hasDiff( sourceRef: string, targetRef: string, ): Promise { await syncGit(); try { return ( (await gitRetry(() => git.diff([sourceRef, targetRef, '--']))) !== '' ); } catch { return true; } } async function handleCommitAuth(localDir: string): Promise { if (!privateKeySet) { await writePrivateKey(); privateKeySet = true; } await configSigningKey(localDir); await writeGitAuthor(); } /** * * Prepare local branch with commit * * 0. Hard reset * 1. Creates local branch with `origin/` prefix * 2. Perform `git add` (respecting mode) and `git remove` for each file * 3. Perform commit * 4. Check whether resulting commit is empty or not (due to .gitignore) * 5. If not empty, return commit info for further processing * */ export async function prepareCommit({ branchName, files, message, force = false, }: CommitFilesConfig): Promise { const localDir = GlobalConfig.get('localDir')!; await syncGit(); logger.debug(`Preparing files for committing to branch ${branchName}`); await handleCommitAuth(localDir); try { await git.reset(ResetMode.HARD); await git.raw(['clean', '-fd']); const parentCommitSha = config.currentBranchSha; await gitRetry(() => git.checkout(['-B', branchName, 'origin/' + config.currentBranch]), ); const deletedFiles: string[] = []; const addedModifiedFiles: string[] = []; const ignoredFiles: string[] = []; for (const file of files) { const fileName = file.path; if (file.type === 'deletion') { try { await git.rm([fileName]); deletedFiles.push(fileName); } catch (err) /* istanbul ignore next */ { const errChecked = checkForPlatformFailure(err); if (errChecked) { throw errChecked; } logger.trace({ err, fileName }, 'Cannot delete file'); ignoredFiles.push(fileName); } } else { if (await isDirectory(upath.join(localDir, fileName))) { // This is usually a git submodule update logger.trace({ fileName }, 'Adding directory commit'); } else if (file.contents === null) { continue; } else { let contents: Buffer; // istanbul ignore else if (typeof file.contents === 'string') { contents = Buffer.from(file.contents); } else { contents = file.contents; } // some file systems including Windows don't support the mode // so the index should be manually updated after adding the file if (file.isSymlink) { await fs.symlink(file.contents, upath.join(localDir, fileName)); } else { await fs.outputFile(upath.join(localDir, fileName), contents, { mode: file.isExecutable ? 0o777 : 0o666, }); } } try { // istanbul ignore next const addParams = fileName === configFileNames[0] ? ['-f', fileName] : fileName; await git.add(addParams); if (file.isExecutable) { await git.raw(['update-index', '--chmod=+x', fileName]); } addedModifiedFiles.push(fileName); } catch (err) /* istanbul ignore next */ { if ( !err.message.includes( 'The following paths are ignored by one of your .gitignore files', ) ) { throw err; } logger.debug(`Cannot commit ignored file: ${fileName}`); ignoredFiles.push(file.path); } } } const commitOptions: Options = {}; if (getNoVerify().includes('commit')) { commitOptions['--no-verify'] = null; } const commitRes = await git.commit(message, [], commitOptions); if ( commitRes.summary && commitRes.summary.changes === 0 && commitRes.summary.insertions === 0 && commitRes.summary.deletions === 0 ) { logger.warn({ commitRes }, 'Detected empty commit - aborting git push'); return null; } logger.debug( { deletedFiles, ignoredFiles, result: commitRes }, `git commit`, ); if (!force && !(await hasDiff('HEAD', `origin/${branchName}`))) { logger.debug( { branchName, deletedFiles, addedModifiedFiles, ignoredFiles }, 'No file changes detected. Skipping commit', ); return null; } const commitSha = ( await git.revparse([branchName]) ).trim() as LongCommitSha; const result: CommitResult = { parentCommitSha, commitSha, files: files.filter((fileChange) => { if (fileChange.type === 'deletion') { return deletedFiles.includes(fileChange.path); } return addedModifiedFiles.includes(fileChange.path); }), }; return result; } catch (err) /* istanbul ignore next */ { return handleCommitError(err, branchName, files); } } export async function pushCommit({ sourceRef, targetRef, files, }: PushFilesConfig): Promise { await syncGit(); logger.debug(`Pushing refSpec ${sourceRef}:${targetRef ?? sourceRef}`); let result = false; try { const pushOptions: TaskOptions = { '--force-with-lease': null, '-u': null, }; if (getNoVerify().includes('push')) { pushOptions['--no-verify'] = null; } const pushRes = await gitRetry(() => git.push('origin', `${sourceRef}:${targetRef ?? sourceRef}`, pushOptions), ); delete pushRes.repo; logger.debug({ result: pushRes }, 'git push'); incLimitedValue('Commits'); result = true; } catch (err) /* istanbul ignore next */ { handleCommitError(err, sourceRef, files); } return result; } export async function fetchBranch( branchName: string, ): Promise { await syncGit(); logger.debug(`Fetching branch ${branchName}`); try { const ref = `refs/heads/${branchName}:refs/remotes/origin/${branchName}`; await gitRetry(() => git.pull(['origin', ref, '--force'])); const commit = (await git.revparse([branchName])).trim() as LongCommitSha; config.branchCommits[branchName] = commit; config.branchIsModified[branchName] = false; return commit; } catch (err) /* istanbul ignore next */ { return handleCommitError(err, branchName); } } export async function commitFiles( commitConfig: CommitFilesConfig, ): Promise { try { const commitResult = await prepareCommit(commitConfig); if (commitResult) { const pushResult = await pushCommit({ sourceRef: commitConfig.branchName, files: commitConfig.files, }); if (pushResult) { const { branchName } = commitConfig; const { commitSha } = commitResult; config.branchCommits[branchName] = commitSha; config.branchIsModified[branchName] = false; return commitSha; } } return null; } catch (err) /* istanbul ignore next */ { if (err.message.includes('[rejected] (stale info)')) { throw new Error(REPOSITORY_CHANGED); } throw err; } } export function getUrl({ protocol, auth, hostname, host, repository, }: { protocol?: GitProtocol; auth?: string; hostname?: string; host?: string; repository: string; }): string { if (protocol === 'ssh') { // TODO: types (#22198) return `git@${hostname!}:${repository}.git`; } return URL.format({ protocol: protocol ?? 'https', auth, hostname, host, pathname: repository + '.git', }); } let remoteRefsExist = false; /** * * Non-branch refs allow us to store git objects without triggering CI pipelines. * It's useful for API-based branch rebasing. * * @see https://stackoverflow.com/questions/63866947/pushing-git-non-branch-references-to-a-remote/63868286 * */ export async function pushCommitToRenovateRef( commitSha: string, refName: string, section = 'branches', ): Promise { const fullRefName = `refs/renovate/${section}/${refName}`; await git.raw(['update-ref', fullRefName, commitSha]); await git.push(['--force', 'origin', fullRefName]); remoteRefsExist = true; } /** * * Removes all remote "refs/renovate/*" refs in two steps: * * Step 1: list refs * * $ git ls-remote origin "refs/renovate/*" * * > cca38e9ea6d10946bdb2d0ca5a52c205783897aa refs/renovate/foo * > 29ac154936c880068994e17eb7f12da7fdca70e5 refs/renovate/bar * > 3fafaddc339894b6d4f97595940fd91af71d0355 refs/renovate/baz * > ... * * Step 2: * * $ git push --delete origin refs/renovate/foo refs/renovate/bar refs/renovate/baz * * If Step 2 fails because the repo doesn't allow bulk changes, we'll remove them one by one instead: * * $ git push --delete origin refs/renovate/foo * $ git push --delete origin refs/renovate/bar * $ git push --delete origin refs/renovate/baz */ export async function clearRenovateRefs(): Promise { if (!gitInitialized || !remoteRefsExist) { return; } logger.debug(`Cleaning up Renovate refs: refs/renovate/*`); const renovateRefs: string[] = []; const obsoleteRefs: string[] = []; try { const rawOutput = await git.listRemote([config.url, 'refs/renovate/*']); const refs = rawOutput .split(newlineRegex) .map((line) => line.replace(regEx(/[0-9a-f]+\s+/i), '').trim()) .filter((line) => line.startsWith('refs/renovate/')); renovateRefs.push(...refs); } catch (err) /* istanbul ignore next */ { logger.warn({ err }, `Renovate refs cleanup error`); } const nonSectionedRefs = renovateRefs.filter((ref) => { return ref.split('/').length === 3; }); obsoleteRefs.push(...nonSectionedRefs); const renovateBranchRefs = renovateRefs.filter((ref) => ref.startsWith('refs/renovate/branches/'), ); obsoleteRefs.push(...renovateBranchRefs); if (obsoleteRefs.length) { try { const pushOpts = ['--delete', 'origin', ...obsoleteRefs]; await git.push(pushOpts); } catch (err) { /* istanbul ignore else */ if (bulkChangesDisallowed(err)) { for (const ref of obsoleteRefs) { try { const pushOpts = ['--delete', 'origin', ref]; await git.push(pushOpts); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'Error deleting obsolete refs'); break; } } } else { logger.warn({ err }, 'Error deleting obsolete refs'); } } } remoteRefsExist = false; } const treeItemRegex = regEx( /^(?\d{6})\s+(?blob|tree|commit)\s+(?[0-9a-f]{40})\s+(?.*)$/, ); const treeShaRegex = regEx(/tree\s+(?[0-9a-f]{40})\s*/); /** * * Obtain top-level items of commit tree. * We don't need subtree items, so here are 2 steps only. * * Step 1: commit SHA -> tree SHA * * $ git cat-file -p * * > tree * > parent 59b8b0e79319b7dc38f7a29d618628f3b44c2fd7 * > ... * * Step 2: tree SHA -> tree items (top-level) * * $ git cat-file -p * * > 040000 tree 389400684d1f004960addc752be13097fe85d776 src * > ... * > 100644 blob 7d2edde437ad4e7bceb70dbfe70e93350d99c98b package.json * */ export async function listCommitTree( commitSha: LongCommitSha, ): Promise { const commitOutput = await git.catFile(['-p', commitSha]); const { treeSha } = treeShaRegex.exec(commitOutput)?.groups ?? /* istanbul ignore next: will never happen */ {}; const contents = await git.catFile(['-p', treeSha]); const lines = contents.split(newlineRegex); const result: TreeItem[] = []; for (const line of lines) { const matchGroups = treeItemRegex.exec(line)?.groups; if (matchGroups) { const { path, mode, type, sha } = matchGroups; result.push({ path, mode, type, sha: sha as LongCommitSha }); } } return result; }