diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index 95ccf8f8f7..edf7a7c503 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -175,6 +175,12 @@ public GitClient subGit(String subdir) { return Git.USE_CLI ? super.subGit(subdir) : jgit.subGit(subdir); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); + } + /** {@inheritDoc} */ @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index ff07100eed..c650138c4a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -351,6 +351,12 @@ public GitClient subGit(String subdir) { return new CliGitAPIImpl(gitExe, new File(workspace, subdir), listener, environment); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new CliGitAPIImpl(gitExe, new File(somedir), listener, environment); + } + /** * Initialize an empty repository for further git operations. * @@ -837,33 +843,56 @@ public void execute() throws GitException, InterruptedException { } if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - } else if (!referencePath.isDirectory()) { - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); + } + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { + listener.getLogger() + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); + if (!referencePath.getPath().equals(reference)) { + // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path '" + reference + "' replaced with: '" + + referencePath.getPath() + "'"; + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] " + msg); + } else { + listener.getLogger().println("[WARNING] " + msg + " does not exist"); + } + reference = referencePath.getPath(); } - if (!objectsPath.isDirectory()) { - listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (not a git repo?): " - + objectsPath); + + if (!referencePath.exists()) { + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + } else if (!referencePath.isDirectory()) { + listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); } else { - File alternates = new File(workspace, ".git/objects/info/alternates"); - try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset())) { - String absoluteReference = - objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - w.print(absoluteReference); - } catch (IOException e) { - listener.error("Failed to setup reference"); + File objectsPath = getObjectsFile(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) { + listener.getLogger() + .println("[WARNING] Reference path does not contain an objects directory " + + "(not a git repo?): " + objectsPath); + } else { + // Go behind git's back to write a meta file in new workspace + File alternates = new File(workspace, ".git/objects/info/alternates"); + try (PrintWriter w = new PrintWriter( + alternates, Charset.defaultCharset().toString())) { + String absoluteReference = + objectsPath.getAbsolutePath().replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); + // git implementations on windows also use + w.print(absoluteReference); + } catch (UnsupportedEncodingException ex) { + listener.error("Default character set is an unsupported encoding"); + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); + } } } } @@ -1494,14 +1523,16 @@ public void execute() throws GitException, InterruptedException { } } if ((ref != null) && !ref.isEmpty()) { - File referencePath = new File(ref); - if (!referencePath.exists()) { - listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); - } else if (!referencePath.isDirectory()) { - listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - } else { - args.add("--reference", ref); - } + if (!isParameterizedReferenceRepository(ref)) { + File referencePath = new File(ref); + if (!referencePath.exists()) { + listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); + } else if (!referencePath.isDirectory()) { + listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); + } else { + args.add("--reference", ref); + } + } // else handled below in per-module loop } if (shallow) { if (depth == null) { @@ -1550,9 +1581,34 @@ public void execute() throws GitException, InterruptedException { listener.error("Invalid repository for " + sModuleName); throw new GitException("Invalid repository for " + sModuleName); } + String strURIish = urIish.toPrivateString(); + + if (isParameterizedReferenceRepository(ref)) { + File referencePath = findParameterizedReferenceRepository(ref, strURIish); + if (referencePath == null) { + listener.getLogger() + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + ref); + } else { + String expRef = null; + if (referencePath.getPath().equals(ref)) { + expRef = ref; + } else { + expRef = referencePath.getPath(); + expRef += " (expanded from " + ref + ")"; + } + if (!referencePath.exists()) { + listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); + } else if (!referencePath.isDirectory()) { + listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); + } else { + args.add("--reference", referencePath.getPath()); + } + } + } // Find credentials for this URL - StandardCredentials cred = credentials.get(urIish.toPrivateString()); + StandardCredentials cred = credentials.get(strURIish); if (parentCredentials) { String parentUrl = getRemoteUrl(getDefaultRemote()); URIish parentUri = null; @@ -1662,6 +1718,45 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr return StringUtils.trim(firstLine(result)); } + /** {@inheritDoc} */ + @Override + public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { + return getRemoteUrlMap(".url="); + } + + /** {@inheritDoc} */ + @Override + public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { + return getRemoteUrlMap(".pushurl="); + } + + private Map getRemoteUrlMap(String configKeySuffix) throws GitException, InterruptedException { + String result = launchCommand("config", "--local", "--list"); + Map uriNames = new HashMap<>(); + for (String line : result.split("\\R+")) { + line = StringUtils.trim(line); + if (!line.startsWith("remote.") || !line.contains(configKeySuffix)) { + continue; + } + + String remoteName = StringUtils.substringBetween(line, "remote.", configKeySuffix); + String remoteUri = StringUtils.substringAfter(line, configKeySuffix); + + // If uri String values end up identical, Map only stores one entry + uriNames.put(remoteUri, remoteName); + + try { + URI u = new URI(remoteUri); + uriNames.put(u.toASCIIString(), remoteName); + URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); + uriNames.put(uSafe.toString(), remoteName); + uriNames.put(uSafe.toASCIIString(), remoteName); + } catch (URISyntaxException ue) { + } // ignore, move along + } + return uriNames; + } + /** {@inheritDoc} */ @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { @@ -2849,8 +2944,16 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e } if (status != 0) { - throw new GitException("Command \"" + command + "\" returned status code " + status + ":\nstdout: " - + stdout + "\nstderr: " + stderr); + if (workDir == null) + workDir = java.nio.file.Paths.get(".") + .toAbsolutePath() + .normalize() + .toFile(); + throw new GitException("Command \"" + command + + "\" executed in workdir \"" + workDir.toString() + + "\" returned status code " + status + + ":\nstdout: " + stdout + + "\nstderr: " + stderr); } return stdout; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 97db99d055..f0851bb058 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -221,6 +221,28 @@ public interface GitClient { */ String getRemoteUrl(String name) throws GitException, InterruptedException; + /** + * getRemoteUrls. + * + * @return a Map where String keys represent URIs (with and + * without passwords, if any; ASCII or not, if + * applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + Map getRemoteUrls() throws GitException, InterruptedException; + + /** + * getRemotePushUrls. + * + * @return a Map where String keys represent push-only URIs + * (with and without passwords, if any; ASCII or not, + * if applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + Map getRemotePushUrls() throws GitException, InterruptedException; + /** * For a given repository, set a remote's URL * @@ -696,12 +718,23 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat */ List revList(String ref) throws GitException, InterruptedException; + // --- new instance of same applied class + + /** + * newGit. + * + * @return an {@link IGitAPI} implementation to manage another git repository + * with same general settings and implementation as the current one. + * @param somedir a {@link java.lang.String} object. + */ + GitClient newGit(String somedir); + // --- submodules /** * subGit. * - * @return a IGitAPI implementation to manage git submodule repository + * @return an {@link IGitAPI} implementation to manage git submodule repository * @param subdir a {@link java.lang.String} object. */ GitClient subGit(String subdir); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 94e63ed330..dc765990bc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -55,6 +55,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; @@ -338,6 +339,12 @@ public GitClient subGit(String subdir) { return new JGitAPIImpl(new File(workspace, subdir), listener); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new JGitAPIImpl(new File(somedir), listener); + } + /** {@inheritDoc} */ @Override public void setAuthor(String name, String email) throws GitException { @@ -1149,6 +1156,39 @@ public String getRemoteUrl(String name) throws GitException { } } + /** {@inheritDoc} */ + @Override + public Map getRemoteUrls() throws GitException, InterruptedException { + return collectRemoteUris(RemoteConfig::getURIs); + } + + /** {@inheritDoc} */ + @Override + public Map getRemotePushUrls() throws GitException, InterruptedException { + return collectRemoteUris(RemoteConfig::getPushURIs); + } + + private Map collectRemoteUris(Function> uriSelector) + throws GitException { + Map uriNames = new HashMap<>(); + try (Repository repo = getRepository()) { + Config c = repo.getConfig(); + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { + String remoteName = rc.getName(); + for (URIish u : uriSelector.apply(rc)) { + // If uri String values end up identical, Map only stores one entry + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException ue) { + throw new GitException(ue.toString()); + } + return uriNames; + } + /** * getRepository. * @@ -1679,37 +1719,61 @@ public void execute() throws GitException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - } else if (!referencePath.isDirectory()) { + // Note: keep in sync with similar logic in CliGitAPIImpl.java + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, + // but not into build console via listener + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); + } + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { listener.getLogger() - .println("[WARNING] Reference path is not a directory: " + reference); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); + if (!referencePath.getPath().equals(reference)) { + // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path '" + reference + "' replaced with: '" + + referencePath.getPath() + "'"; + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] " + msg); + } else { + listener.getLogger().println("[WARNING] " + msg + " does not exist"); + } + reference = referencePath.getPath(); } - if (!objectsPath.isDirectory()) { + if (!referencePath.exists()) { + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + reference); + } else if (!referencePath.isDirectory()) { listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (no git repo?): " - + objectsPath); + .println("[WARNING] Reference path is not a directory: " + reference); } else { - try { - File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = - objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { - // git implementations on windows also use - w.print(absoluteReference); + // reference path can either be a normal or a base repository + File objectsPath = getObjectsFile(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) { + listener.getLogger() + .println( + "[WARNING] Reference path does not contain an objects directory (no git repo?): " + + objectsPath); + } else { + // Go behind git's back to write a meta file in new workspace + try { + File alternates = new File(workspace, ".git/objects/info/alternates"); + String absoluteReference = objectsPath + .getAbsolutePath() + .replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); + try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { + // git implementations on windows also use + w.print(absoluteReference); + } + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); } - } catch (FileNotFoundException e) { - listener.error("Failed to setup reference"); } } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 45b17e614f..573dc68889 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -3,6 +3,7 @@ import static java.util.Arrays.copyOfRange; import static org.apache.commons.lang3.StringUtils.join; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; @@ -10,12 +11,28 @@ import hudson.plugins.git.Revision; import hudson.plugins.git.Tag; import hudson.remoting.Channel; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import org.apache.commons.codec.digest.DigestUtils; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -35,6 +52,7 @@ */ abstract class LegacyCompatibleGitAPIImpl extends AbstractGitAPIImpl implements IGitAPI { + private static final Logger LOGGER = Logger.getLogger(LegacyCompatibleGitAPIImpl.class.getName()); private HostKeyVerifierFactory hostKeyFactory; /** @@ -181,6 +199,1251 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** For referenced directory check if it is a full or bare git repo + * and return the File object for its "objects" sub-directory. + * (Note that for submodules and other cases with externalized Git + * metadata, the "objects" directory may be NOT under "reference"). + * If there is nothing to find, or inputs are bad, returns null. + * The idea is that checking for null allows to rule out non-git + * paths, while a not-null return value is instantly usable by + * some code which plays with git under its hood. + */ + public static File getObjectsFile(String reference) { + if (reference == null || reference.isEmpty()) { + return null; + } + return getObjectsFile(new File(reference)); + } + + public static File getObjectsFile(File reference) { + // reference pathname can either point to a "normal" workspace + // checkout or a bare repository + + if (reference == null) { + return reference; + } + + if (!reference.exists()) { + return null; + } + + if (!reference.isDirectory()) { + return null; + } + + File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those + File objects = null; + + if (fGit.exists()) { + if (fGit.isDirectory()) { + objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below + /* + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null + return objects; // Some Java error, could not make object from the paths involved + } + */ + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is a directory"); + } else { + // If ".git" FS object exists and is a not-empty file (and + // is not a dir), then its content may point to some other + // filesystem location for the Git-internal data. + // For example, a checked-out submodule workspace can point + // to the index and other metadata stored in its "parent" + // repository's directory: + // "gitdir: ../.git/modules/childRepoName" + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is NOT a directory"); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8"))) { + String line; + while ((line = reader.readLine()) != null) { + String[] parts = line.split(":", 2); + if (parts.length >= 2) { + String key = parts[0].trim(); + String value = parts[1].trim(); + if (key.equals("gitdir")) { + // value is the gitdir path (e.g. "../../.git/modules/sub"); + // we need the "objects" subdirectory *inside* that gitdir, + // not the gitdir directory itself. + objects = new File(new File(reference, value), "objects"); + LOGGER.log( + Level.FINE, + "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath() + + "', found reference to objects which should be at: '" + + objects.getAbsolutePath() + "'"); + // Note: we don't use getCanonicalPath() here to avoid further filesystem + // access and possible exceptions (the getAbsolutePath() is about string + // processing), but callers might benefit from canonicising and ensuring + // unique pathnames (for equality checks etc.) with relative components + // and symlinks resolved. + // On another hand, keeping the absolute paths, possibly, relative to a + // parent directory as prefix, allows callers to match/compare such parent + // prefixes for the contexts the callers would define for themselves. + break; + } + LOGGER.log( + Level.FINEST, + "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath() + + "', ignoring line: " + line); + } + } + if (objects == null) { + LOGGER.log( + Level.WARNING, + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + + "': did not contain a 'gitdir:' entry"); + } + } catch (IOException e) { + LOGGER.log( + Level.SEVERE, + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + "': " + e.toString()); + objects = null; + } + } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + fGit.getAbsolutePath() + "'"); + } + + if (objects == null || !objects.isDirectory()) { + // either reference path is bare repo (no ".git" inside), + // or we have failed interpreting ".git" contents above + objects = new File(reference, "objects"); // bare + /* + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } + */ + // This clause below is redundant for production, but useful for troubleshooting + if (objects.exists()) { + if (objects.isDirectory()) { + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + "' which is a directory"); + } else { + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + + "' which is NOT a directory"); + } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + objects.getAbsolutePath() + "'"); + } + } + + if (!objects.exists()) { + return null; + } + + if (!objects.isDirectory()) { + return null; + } + + // If we get here, we have a non-null File referencing a + // "(.git/)objects" subdir under original referencePath + return objects; + } + + /** Get remote name/URL pairs for the git repository at {@code dirname} + * (bare or not, resolved the same way as getObjectsFile()) as fast as + * possible: first try parsing its "config" file directly in memory (no + * subprocess, no JGit Repository object), and only if that is not + * possible or not safe (see {@link #tryReadRemoteUrlsFromConfigFile}), + * fall back to a normal GitClient (CliGit forks `git config --local + * --list`; JGit opens a full Repository) - same as the rest of this + * class did before this fast path was added. + * + *

This is used while walking many candidate directories in + * getSubmodulesUrls(), where most of the cost is exactly this per-dir + * remote lookup. + * + * @return a Map of remote URL to remote name, same shape as + * {@link GitClient#getRemoteUrls()} + */ + private Map getRemoteUrlsFast(String dirname) throws GitException, InterruptedException { + Map uriNames = tryReadRemoteUrlsFromConfigFile(dirname); + if (uriNames != null) { + return uriNames; + } + // "this" during a checkout typically represents the job workspace, + // but here we look at some reference/cache repository elsewhere, + // with the same implementation as the end-user set up (CliGit/jGit) + return this.newGit(dirname).getRemoteUrls(); + } + + /** Try to read remote name/URL pairs directly out of the "config" file + * of the git repository at {@code dirname} (bare or not; resolved to the + * actual GIT_DIR the same way {@link #getObjectsFile} does, so this also + * works for a checked-out submodule's ".git" gitlink file), without + * spawning a git subprocess or opening a full JGit {@link Repository}. + * + * @return a Map of remote URL to remote name (several URL variants per + * remote, mirroring what {@link GitClient#getRemoteUrls()} + * implementations already return, to keep needle matching + * elsewhere in this class equally likely to hit), or + * {@code null} if the config file is missing, unreadable, + * unparseable, or uses {@code include}/{@code includeIf} + * directives that could define remotes elsewhere and + * which we do not attempt to resolve here - callers + * should fall back to a full GitClient-based lookup. + */ + private static Map tryReadRemoteUrlsFromConfigFile(String dirname) { + File objects = getObjectsFile(dirname); + if (objects == null) { + return null; + } + File gitDir = objects.getParentFile(); + if (gitDir == null) { + return null; + } + File configFile = new File(gitDir, "config"); + if (!configFile.isFile() || !configFile.canRead()) { + return null; + } + + String text; + try { + text = new String(Files.readAllBytes(configFile.toPath()), StandardCharsets.UTF_8); + } catch (IOException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to read '" + configFile.getAbsolutePath() + "': " + e.toString()); + return null; + } + + // Bail out rather than risk silently missing remotes actually defined + // in a file pulled in this way - we do not resolve includes here. + if (text.toLowerCase(Locale.ENGLISH).contains("[include")) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): '" + configFile.getAbsolutePath() + + "' has an include/includeIf directive, falling back to GitClient instead of parsing it directly"); + return null; + } + + Config config = new Config(); + try { + config.fromText(text); + } catch (ConfigInvalidException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to parse '" + configFile.getAbsolutePath() + "': " + e.toString()); + return null; + } + + Map uriNames = new LinkedHashMap<>(); + try { + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(config)) { + String remoteName = rc.getName(); + for (URIish u : rc.getURIs()) { + // If uri String values end up identical, Map only stores one entry - + // same trade-off existing GitClient.getRemoteUrls() implementations make. + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to interpret remotes from '" + configFile.getAbsolutePath() + "': " + + e.toString()); + return null; + } + return uriNames; + } + + /** Handle magic strings in the reference pathname to sort out patterns + * classified as evaluated by parametrization, as handled below + * + * @param reference Pathname (maybe with magic suffix) to reference repo + */ + public static Boolean isParameterizedReferenceRepository(File reference) { + if (reference == null) { + return false; + } + return isParameterizedReferenceRepository(reference.getPath()); + } + + public static Boolean isParameterizedReferenceRepository(String reference) { + if (reference == null || reference.isEmpty()) { + return false; + } + + return reference.endsWith("/${GIT_URL_SHA256}") + || reference.endsWith("/${GIT_URL_SHA256_FALLBACK}") + || reference.endsWith("/${GIT_URL_BASENAME}") + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") + || reference.endsWith("/${GIT_SUBMODULES}") + || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}"); + } + + /** There are many ways to spell an URL to the same repository even if + * using the same access protocol. This routine converts the "url" string + * in a way that helps us confirm whether two spellings mean same thing. + */ + @SuppressFBWarnings( + value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Path operations below intentionally use absolute '/' in some cases") + public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { + String urlNormalized = + url.replaceAll("/+$", "").replaceAll("\\.git$", "").toLowerCase(Locale.ENGLISH); + if (!url.contains("://")) { + if (!url.startsWith("/") && !url.startsWith(".")) { + // Not an URL with schema, not an absolute or relative pathname + if (checkLocalPaths) { + File urlPath = new File(url); + if (urlPath.exists()) { + try { + // Check if the string in urlNormalized is a valid + // relative path (subdir) in current working directory + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); + } catch (java.nio.file.InvalidPathException ipe1) { + // e.g. Illegal char <:> at index 30: + // C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone + try { + // Re-check in another manner + urlNormalized = "file://" + + Paths.get(Paths.get("", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); + } catch (java.nio.file.InvalidPathException ipe2) { + // Finally, fall back to checking the originally + // fully-qualified path + urlNormalized = "file://" + + Paths.get(Paths.get("/", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); + } + } + } else { + // Also not a subdirectory of current dir without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Assume it is not a path + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Looks like a local path + if (url.startsWith("/")) { + urlNormalized = "file://" + urlNormalized; + } else { + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); + } + } + } + + LOGGER.log( + Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + return urlNormalized; + } + + /** Find referenced URLs in this repo and its submodules (or other + * subdirs with git repos), recursively. Current primary use is for + * parameterized refrepo/${GIT_SUBMODULES} handling. + * + * @return an AbstractMap.SimpleEntry, containing a Boolean to denote + * an exact match (or lack thereof) for the needle (if searched for), + * and a Set of (unique) String lists, each with four elements: + * get(0) directory of nested submodule (relative to current workspace root) + * The current workspace would be listed as directory "" and consumers + * should check these entries last if they care for most-specific hits + * with smaller-scope reference repositories. + * get(1) url as returned by getRemoteUrls() - fetch URLs, maybe several + * entries per remote + * get(2) urlNormalized from normalizeGitUrl(url, true) (local pathnames + * fully qualified) + * get(3) remoteName as defined in that nested submodule + * + * If the returned SimpleEntry has the Boolean flag as False but also + * a Set which is not empty, and a search for "needle" was requested, + * then that Set lists some not-exact matches for existing sub-dirs + * with repositories that seem likely to be close hits (e.g. remotes + * there *probably* point to other URLs of same repo as the needle, + * or its forks - so these directories are more likely than others to + * contain the reference commits needed for the faster git checkouts). + * + * For a search with needle==null, the Boolean flag would be False too, + * and the Set would just detail all found sub-repositories. + * + * @param referenceBaseDir - the reference repository, or container thereof + * @param needle - an URL which (or its normalized variant coming from + * normalizeGitUrl(url, true)) we want to find: + * if it is not null - then stop and return just hits + * for it as soon as we have something. + * @param checkRemotesInReferenceBaseDir - if true (reasonable default for + * external callers), the referenceBaseDir would be added + * to the list of dirs for listing known remotes in search + * for a needle match or for the big listing. Set to false + * when recursing, since this directory was checked already + * as part of parent directory inspection. + * + * If a {@value #GITCACHE_MAP_FILENAME} file is present directly under + * referenceBaseDir, it is consulted first for a needle search: a hit there + * (confirmed still a usable git directory which still actually has a + * remote configured for the needle URL, not just a stale filename + * association) is returned immediately, without probing candidate + * subdirectory names or walking the filesystem. A missing file, + * unreadable file, or lack of a (still valid and verified) matching + * entry falls through to the directory-walk search below, unaffected. + */ + public SimpleEntry>> getSubmodulesUrls( + String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { + // Keep track of where we've already looked in the "result" Set, to + // avoid looking in same places (different strategies below) twice. + // And eventually return this Set or part of it as the answer. + LinkedHashSet> result = new LinkedHashSet<>(); // Retain order of insertion + File f = null; + // Helper list storage in loops below + ArrayList arrDirnames = new ArrayList(); + + // Note: an absolute path is not necessarily the canonical one + // We want to hit same dirs only once, so canonicize paths below + String referenceBaseDirAbs; + try { + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath(); + } catch (IOException e) { + // Access error while dereferencing some parent?.. + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to canonicize '" + + referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + + e.toString()); + // return new SimpleEntry<>(false, result); + } + + // Fast path: if external tooling (e.g. register-git-cache.sh from + // https://github.com/jimklimov/git-refrepo-scripts) maintains a + // GITCACHE_MAP_FILENAME mapping of known remote URLs to their + // subdirectory under this referenceBaseDir, try it before spending + // any I/O on directory probing or `git remote -v` subprocess calls. + if (needle != null && !needle.isEmpty()) { + List mapHit = tryGitCacheMapFile(referenceBaseDirAbs, needle); + if (mapHit != null) { + LinkedHashSet> resultBest = new LinkedHashSet<>(); + resultBest.add(mapHit); + return new SimpleEntry<>(true, resultBest); + } + } + + // "this" during a checkout typically represents the job workspace, + // but we want to inspect the reference repository located elsewhere + // with the same implementation as the end-user set up (CliGit/jGit) + GitClient referenceGit = this.newGit(referenceBaseDirAbs); + + Boolean isBare = false; + try { + isBare = ((hudson.plugins.git.IGitAPI) referenceGit).isBareRepository(); + } catch (InterruptedException | GitException e) { + // Proposed base directory whose subdirs contain refrepos might + // itself be not a repo. Shouldn't be per reference scripts, but... + if (e.toString().contains("GIT_DISCOVERY_ACROSS_FILESYSTEM")) { + // Note the message may be localized, envvar name should not be: + // stderr: fatal: not a git repository (or any parent up to mount point /some/path) + // Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). + // As far as the logic below is currently concerned, we do not + // look for submodules directly in a bare repo. + isBare = true; + } else { + // Some other error + isBare = false; // At least try to look into submodules... + } + + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to determine isBareRepository() in '" + + referenceBaseDirAbs + "'; " + "assuming '" + isBare + "': " + + e.toString()); + } + + // Simplify checks below by stating a useless needle is null + if (needle != null && needle.isEmpty()) { + needle = null; + } + // This is only used and populated if needle is not null + String needleNorm = null; + + // This is only used and populated if needle is not null, and + // can be used in the end to filter not-exact match suggestions + String needleBasename = null; + String needleBasenameLC = null; + String needleNormBasename = null; + String needleSha = null; + + // If needle is not null, first look perhaps in the subdir(s) named + // with base-name of the URL with and without a ".git" suffix, then + // in SHA256 named dir that can match it; note that this use-case + // might pan out also if "this" repo is bare and can not have "proper" + // git submodules - but was prepared for our other options. + if (needle != null) { + int sep = needle.lastIndexOf("/"); + if (sep < 0) { + needleBasename = needle; + } else { + needleBasename = needle.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll("\\.[Gg][Ii][Tt]$", ""); + + needleNorm = normalizeGitUrl(needle, true); + sep = needleNorm.lastIndexOf("/"); + if (sep < 0) { + needleNormBasename = needleNorm; + } else { + needleNormBasename = needleNorm.substring(sep + 1); + } + needleNormBasename = needleNormBasename.replaceAll("\\.git$", ""); + + // Try with the basename without .git extension, and then with one. + // First we try the caller-provided string casing, then normalized. + // Note that only after this first smaller pass which we hope to + // succeed quickly, we engage in heavier (by I/O and computation) + // investigation of submodules, and then similar loop against any + // remaining direct subdirs that contain a ".git" (or "objects") + // FS object. + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); + needleBasenameLC = needleBasename.toLowerCase(Locale.ENGLISH); + if (!needleBasenameLC.equals(needleBasename)) { + // Retry with lowercased dirname + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC + ".git"); + } + if (!needleNormBasename.equals(needleBasenameLC)) { + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename + ".git"); + } + + needleSha = DigestUtils.sha256Hex(needleNorm); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); + + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + + "', per arrDirnames: " + arrDirnames.toString()); + + for (String dirname : arrDirnames) { + f = new File(dirname); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); + if (getObjectsFile(f) != null) { + try { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking for submodule URL needle='" + needle + + "' in existing abs refrepo subdir '" + dirname + "'"); + // Prefer parsing the "config" file directly (no subprocess, + // no JGit Repository object) over going through newGit() + // (dirname is already absolute anyway, and going through + // referenceGit.subGit() may lose the disk letter on Windows + // agents) - see getRemoteUrlsFast() for when it falls back. + Map uriNames = getRemoteUrlsFast(dirname); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): getRemoteUrlsFast() returned this Map uriNames: " + + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + // Best hit, caller really only needs to look at this one + // (as the first entry in the returned ordered set) + LinkedHashSet> result_best = + new LinkedHashSet<>(); // Retain order of insertion + result_best.add(List.of(dirname, uri, uriNorm, remoteName)); + result_best.addAll(result); + return new SimpleEntry<>(true, result_best); + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(List.of(dirname, uri, uriNorm, remoteName)); + } + } catch (Throwable t) { + // ignore, go to next slide + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); + } + } + } + } // if needle, look in basename-like and SHA dirs first + + // Needle or not, the rest of directory walk to collect data is the + // same, so follow a list of whom we want to visit in likely-quickest + // hit order. Note that if needle is null, we walk over everything + // that makes sense to visit, to return info on all git remotes found + // in or under this directory; however if it is not-null, we still + // try to have minimal overhead to complete as soon as we match it. + // TODO: Refactor to avoid lookups of dirs that may prove not needed + // in the end (aim for less I/Os to find the goal)... or is one dir + // listing a small price to pay for maintaining one unified logic? + + // If current dir does have submodules, first dig into submodules, + // when there is no deeper to drill, report remote URLs and step + // back from recursion. This way we have least specific repo last, + // if several have the replica (assuming the first hits are smaller + // scopes). + + // Track where we have looked already; note that values in result[] + // (if any from needle-search above) are absolute pathnames + LinkedHashSet checkedDirs = new LinkedHashSet<>(); + for (List resultEntry : result) { + checkedDirs.add(resultEntry.get(0)); + } + + // TODO: If current repo is NOT bare, also check git submodules registered + // in .gitmodules for a faster possible answer (MODNAME.{url,path} mapping). + // See git commit 9cff320a (a couple commits before this line was added) + // where a commented-away exploration of how this might be done was removed. + // Note that git does not really support submodule definitions with several + // URLs mapped to same name. Its `git fsck` can complain and "fix" those + // discrepancies, and other operations may look at just some one hit. + + // A different faster-answer idea (external tooling-maintained mapping + // of URL to subdirectory, checked at the top of this method before we + // even get here) is implemented via tryGitCacheMapFile() and does not + // require this repo to have real git submodules at all. + + // If current repo *is* bare (can't have proper submodules), or if the + // end-users just cloned or linked some more repos into this container, + // follow up with direct child dirs that have a ".git" (or "objects") + // FS object inside: + + // Check subdirs that are git workspaces; note that values in checkedDirs + // are absolute pathnames. If we did look for the needle, array already + // starts with some "prioritized" pathnames which we should not directly + // inspect again... but should recurse into first anyhow. + File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); + if (directories != null) { + // listFiles() "...returns null if this abstract pathname + // does not denote a directory, or if an I/O error occurs" + for (File dir : directories) { + if (getObjectsFile(dir) != null) { + String dirname = dir.getPath().replaceAll("/+$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } + } + } + } + + // Finally check pattern's parent dir + // * Look at remote URLs in current dir after the guessed subdirs failed, + // and return then. + if (checkRemotesInReferenceBaseDir) { + if (getObjectsFile(referenceBaseDirAbs) != null) { + arrDirnames.add(referenceBaseDirAbs); + } + } + + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at " + + ((isBare ? "" : "submodules first, then ")) + + "all subdirs that have a .git, under refrepo '" + + referenceBaseDirAbs + "' per absolute arrDirnames: " + + arrDirnames.toString()); + + for (String dirname : arrDirnames) { + // Note that here dirnames deal in absolutes + f = new File(dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); + if (f.exists() && f.isDirectory()) { + // No checks for ".git" or "objects" this time, already checked above + // by getObjectsFile(). Probably should not check exists/dir either, + // but better be on the safe side :) + if (!checkedDirs.contains(dirname)) { + try { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking " + + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + + "in existing refrepo dir '" + dirname + "'"); + Map uriNames = getRemoteUrlsFast(dirname); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): getRemoteUrlsFast() returned this Map uriNames: " + + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); + if (needle != null + && needleNorm != null + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { + // Best hit, caller really only needs to look at this one + // (as the first entry in the returned ordered set) + LinkedHashSet> result_best = + new LinkedHashSet<>(); // Retain order of insertion + result_best.add(List.of(dirname, uri, uriNorm, remoteName)); + result_best.addAll(result); + return new SimpleEntry<>(true, result_best); + } + // Cache the finding to return eventually, for each remote: + // * absolute dirname of a Git workspace + // * original remote URI from that workspace's config + // * normalized remote URI + // * name of the remote from that workspace's config ("origin" etc) + result.add(List.of(dirname, uri, uriNorm, remoteName)); + } + } catch (Throwable t) { + // ignore, go to next slide + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); + } + } + + // Here is a good spot to recurse this routine into a + // subdir that is already a known git workspace, to + // add its data to list and/or return a found needle. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + SimpleEntry>> subEntriesRet = + getSubmodulesUrls(dirname, needle, false); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet> subEntries = subEntriesRet.getValue(); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + + subEntries.size() + " found mappings"); + if (!subEntries.isEmpty()) { + if (needle != null && subEntriesExactMatched) { + // We found nothing... until now! Bubble it up! + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + + "': " + subEntries.iterator().next().get(0)); + return subEntriesRet; + } + // ...else collect results to inspect and/or propagate later + result.addAll(subEntries); + } + } + } + + // Nothing found, if we had a needle - so report there are no hits + // If we did not have a needle, we did not search for it - return + // below whatever result we have, then. + if (needle != null) { + if (result.size() == 0) { + // Completely nothing git-like found here, return quickly + return new SimpleEntry<>(false, result); + } + + // Handle suggestions (not-exact matches) if something from + // results looks like it is related to the needle. + LinkedHashSet> resultFiltered = new LinkedHashSet<>(); + + // Separate lists by suggestion priority: + // 1) URI basename similarity + // 2) Directory basename similarity + LinkedHashSet> resultFiltered1 = new LinkedHashSet<>(); + LinkedHashSet> resultFiltered2 = new LinkedHashSet<>(); + + LinkedHashSet suggestedDirs = new LinkedHashSet<>(); + for (List resultEntry : result) { + checkedDirs.add(resultEntry.get(0)); + } + + for (List subEntry : result) { + // Iterating to filter suggestions in order of original + // directory-walk prioritization under current reference + String dirName = subEntry.get(0); + String uriNorm = subEntry.get(2); + Integer sep; + String uriNormBasename; + String dirBasename; + + // Match basename of needle vs. a remote tracked by an + // existing git directory (automation-ready normalized URL) + sep = uriNorm.lastIndexOf("/"); + if (sep < 0) { + uriNormBasename = uriNorm; + } else { + uriNormBasename = uriNorm.substring(sep + 1); + } + uriNormBasename = uriNormBasename.replaceAll("\\.git$", ""); + + if (uriNormBasename.equals(needleNormBasename)) { + resultFiltered1.add(subEntry); + } + + if (!suggestedDirs.contains(dirName)) { + // Here just match basename of needle vs. an existing + // sub-git directory base name + suggestedDirs.add(dirName); + + sep = dirName.lastIndexOf("/"); + if (sep < 0) { + dirBasename = dirName; + } else { + dirBasename = dirName.substring(sep + 1); + } + dirBasename = dirBasename.replaceAll("\\.git$", ""); + + if (dirBasename.equals(needleNormBasename)) { + resultFiltered2.add(subEntry); + } + } + } + + // Concatenate suggestions in order of priority. + // Hopefully the Set should deduplicate entries + // if something matched twice :) + resultFiltered.addAll(resultFiltered1); // URLs + resultFiltered.addAll(resultFiltered2); // Dirnames + + // Note: flag is false since matches (if any) are + // not exactly for the Git URL requested by caller + return new SimpleEntry<>(false, resultFiltered); + } + + // Did not look for anything in particular + return new SimpleEntry<>(false, result); + } + + /** Name of an optional mapping file maintained by external tooling (see + * https://github.com/jimklimov/git-refrepo-scripts {@code register-git-cache.sh ls} + * output) directly under a getSubmodulesUrls() referenceBaseDir. + * Each line is tab-separated: remote name, remote URL, and (possibly + * empty, for the top-level referenceBaseDir itself) relative pathname + * of the subdirectory that hosts a git clone with that remote, e.g.: + * 

+     *   repo-1717010012<TAB>ssh://git@example.com/foo/bar.git<TAB>bar.git
+     *
+ * This lets getSubmodulesUrls() resolve a needle URL to its cached + * subdirectory in one file read, instead of probing candidate dirnames + * and running `git remote -v` (a subprocess or JGit call) in each of + * the fanned-out subdirectories. + */ + static final String GITCACHE_MAP_FILENAME = ".gitcache.map"; + + /** Look up {@code needle} in a {@value #GITCACHE_MAP_FILENAME} file + * directly under {@code referenceBaseDirAbs}, if one is present. + * + * @param referenceBaseDirAbs absolute, canonical path to the refrepo + * container directory that might hold the mapping file + * @param needle the (not-null, not-empty) URL we are looking for + * @return a 4-element List as used elsewhere in getSubmodulesUrls() + * (dirname, url, urlNormalized, remoteName) for a hit + * that still resolves to a usable git directory which + * still actually has a remote configured for the needle + * (not just a directory that happens to exist), or + * {@code null} if there is no file, it is unreadable, + * or none of its (still valid) entries match the needle + */ + private List tryGitCacheMapFile(String referenceBaseDirAbs, String needle) { + File mapFile = new File(referenceBaseDirAbs, GITCACHE_MAP_FILENAME); + if (!mapFile.isFile() || !mapFile.canRead()) { + return null; + } + + String needleNorm = normalizeGitUrl(needle, true); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(mapFile), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + String[] cols = line.split("\t", -1); + if (cols.length < 2) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): ignoring malformed " + GITCACHE_MAP_FILENAME + " line: '" + line + + "'"); + continue; + } + String remoteName = cols[0].trim(); + String uri = cols[1].trim(); + String dirRel = cols.length > 2 ? cols[2].trim() : ""; + String uriNorm = normalizeGitUrl(uri, true); + + if (!needleNorm.equals(uriNorm) && !needle.equals(uri)) { + continue; + } + + String dirname = dirRel.isEmpty() ? referenceBaseDirAbs : (referenceBaseDirAbs + "/" + dirRel); + if (getObjectsFile(dirname) == null) { + // Stale entry (dir gone, renamed, or never made) - keep + // looking at remaining lines, then fall back to the walk. + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): " + GITCACHE_MAP_FILENAME + " named '" + dirname + "' (recorded as '" + + remoteName + "') for needle='" + needle + + "' but it is not a usable git dir now; ignoring"); + continue; + } + + // The dir exists and looks like a git repo, but the map file + // could still be stale/misleading (URL changed, remote + // renamed or removed, dir repurposed for another repo since + // the map was last saved) - confirm the needle is actually + // among its configured remotes before trusting this hit. + List verified = verifyGitCacheMapEntry(dirname, needle, needleNorm); + if (verified == null) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): " + GITCACHE_MAP_FILENAME + " named '" + dirname + "' (recorded as '" + + remoteName + "') for needle='" + needle + + "' but it has no matching remote configured now; ignoring"); + continue; + } + + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): resolved needle='" + needle + "' via " + GITCACHE_MAP_FILENAME + + " to dir '" + dirname + "', skipping directory walk"); + return verified; + } + } catch (IOException e) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): failed to read " + GITCACHE_MAP_FILENAME + " at '" + mapFile + "': " + + e.toString()); + } + return null; + } + + /** Confirm that {@code dirname} (already known to be a usable git + * directory) actually has a remote configured whose URL matches + * {@code needle}/{@code needleNorm}, so a {@value #GITCACHE_MAP_FILENAME} + * hit is not trusted purely because the directory happens to still exist. + * + * @return a 4-element List (dirname, url, urlNormalized, remoteName) + * using the remote name actually found in {@code dirname} + * (which may differ from the map file's recorded name, + * e.g. after a manual `git remote rename`), or + * {@code null} if no configured remote there matches + */ + private List verifyGitCacheMapEntry(String dirname, String needle, String needleNorm) { + try { + Map uriNames = getRemoteUrlsFast(dirname); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + return List.of(dirname, uri, uriNorm, pair.getValue()); + } + } + } catch (Throwable t) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): failed to verify " + GITCACHE_MAP_FILENAME + " entry in dir '" + dirname + + "' for needle='" + needle + "': " + t.toString()); + } + return null; + } + + /** See above. With null needle, returns all data we can find under the + * referenceBaseDir tree (can take a while) for the caller to parse */ + public SimpleEntry>> getSubmodulesUrls(String referenceBaseDir) { + return getSubmodulesUrls(referenceBaseDir, null, true); + } + /* Do we need a completely parameter-less variant to look under current + * work dir aka Paths.get("").toAbsolutePath().toString(), or under "this" + * GitClient workspace ?.. */ + + /** Yield the File object for the reference repository local filesystem + * pathname. Note that the provided string may be suffixed with expandable + * tokens which allow to store a filesystem structure of multiple small + * reference repositories instead of a big combined repository, while + * providing a single inheritable configuration string value. Callers + * can check whether the original path was used or mangled into another + * by comparing their "reference" with returned object's File.getName(). + * + * At some point this plugin might also maintain that filesystem structure. + * + * @param reference Pathname (maybe with magic suffix) to reference repo + * @param url URL of the repository being cloned, to help choose a + * suitable parameterized reference repo subdirectory. + */ + public File findParameterizedReferenceRepository(File reference, String url) { + if (reference == null) { + return reference; + } + return findParameterizedReferenceRepository(reference.getPath(), url); + } + + public File findParameterizedReferenceRepository(String reference, String url) { + if (reference == null || reference.isEmpty()) { + return null; + } + + File referencePath = new File(reference); + // For mass-configured jobs, like Organization Folders, which inherit + // a refrepo setting (String) into generated MultiBranch Pipelines and + // leaf jobs made for each repo branch and PR, the code below allows + // us to support parameterized paths, with one string leading to many + // reference repositories fanned out under a common location. + // This also works for legacy jobs using a Git SCM. + + // TODO: Consider a config option whether to populate absent reference + // repos (If the expanded path does not have git repo data right now, + // should we populate it into the location expanded by logic below), + // or update existing ones before pulling commits, and how to achieve + // that. Currently this is something that comments elsewhere in the + // git-client-plugin and/or articles on reference repository setup + // considered to be explicitly out of scope of the plugin. + // Note that this pre-population (or update) is done by caller with + // their implementation of git and site-specific connectivity and + // storage desigh, e.g. in case of a build farm it is more likely + // to be a shared path from a common storage server only readable to + // Jenkins and its agents, so write-operations would be done by helper + // scripts that log into the shared storage server to populate or + // update the reference repositories. Note that users may also + // want to run their own scripts to "populate" reference repos + // as symlinks to existing other repos, to support combined + // repo setup for different URLs pointing to same upstream, + // or storing multiple closely related forks together. + // This feature was developed along with a shell script to manage + // reference repositories, both in original combined-monolith layout, + // and in the subdirectory fanout compatible with plugin code below: + // https://github.com/jimklimov/git-scripts/blob/master/register-git-cache.sh + + // Note: Initially we expect the reference to be a realistic dirname + // with a special suffix to substitute after the logic below, so the + // referencePath for that verbatim funny string should not exist now: + if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { + // Drop the trailing keyword to know the root refrepo dirname + String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); + + File referenceBaseDirFile = new File(referenceBaseDir); + if (!referenceBaseDirFile.exists()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); + return null; + } + if (!referenceBaseDirFile.isDirectory()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); + return null; + } + + // Note: this normalization might crush several URLs into one, + // and as far as is known this would be the goal - people tend + // to use or omit .git suffix, or spell with varied case, while + // it means the same thing to known Git platforms, except local + // dirs on case-sensitive filesystems. + // The actual reference repository directory may choose to have + // original URL strings added as remotes (in case some are case + // sensitive and different). + String urlNormalized = normalizeGitUrl(url, true); + + // Let users know why there are many "git config --list" lines in their build log: + LOGGER.log( + Level.INFO, + "Trying to resolve parameterized Git reference repository '" + reference + + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); + + String referenceExpanded = null; + if (reference.endsWith("/${GIT_URL_SHA256}")) { + // This may be the more portable solution with regard to filesystems + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", DigestUtils.sha256Hex(urlNormalized)); + } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { + // The safest option - fall back to parent directory if + // the expanded one does not have git repo data right now: + // it allows to gradually convert a big combined reference + // repository into smaller chunks without breaking builds. + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", DigestUtils.sha256Hex(urlNormalized)); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } + } else if (reference.endsWith("/${GIT_URL_BASENAME}") + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + // This may be the more portable solution with regard to filesystems + // First try with original user-provided casing of the URL (if local + // dirs were cloned manually) + int sep = url.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = url; + } else { + needleBasename = url.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll("\\.git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", Matcher.quoteReplacement(needleBasename)); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); + if (url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory (only if we do not check urlNormalized separately below) + referenceExpanded = referenceBaseDir; + } + } + + if (!url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { + // Retry with automation-ready normalized URL + sep = urlNormalized.lastIndexOf("/"); + if (sep < 0) { + needleBasename = urlNormalized; + } else { + needleBasename = urlNormalized.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll("\\.git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", Matcher.quoteReplacement(needleBasename)); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); + if (getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } + } + } + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + // Here we employ git submodules - so we can reliably match + // remote URLs (easily multiple) to particular modules, and + // yet keep separate git index directories per module with + // smaller scopes - much quicker to check out from than one + // huge combined repo. It would also be much more native to + // tools and custom scriptware that can be involved. + // Beside git-submodule parsing (that only points to one URL + // at a time) his also covers a search for subdirectories + // that host a git repository whose remotes match the URL, + // to handle co-hosting of several remotes (different URLs + // to same repository, e.g. SSH and HTTPS; mirrors; forks). + + // Assuming the provided "reference" directory already hosts + // submodules, we use git tools to find the one subdir which + // has a registered remote URL equivalent (per normalization) + // to the provided "url". + + // Note: we pass the unmodified "url" value here, the routine + // differentiates original spelling vs. normalization while + // looking for its needle in the haystack. + SimpleEntry>> subEntriesRet = + getSubmodulesUrls(referenceBaseDir, url, true); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet> subEntries = subEntriesRet.getValue(); + if (!subEntries.isEmpty()) { + // Normally we should only have one entry here, as sorted + // by the routine, and prefer that first option if a new + // reference repo would have to be made (and none exists). + // If several entries are present after all, iterate until + // first existing hit and return the first entry otherwise. + if (!subEntriesExactMatched) { // else look at first entry below + for (List subEntry : subEntries) { + if (getObjectsFile(subEntry.get(0)) != null + || getObjectsFile(subEntry.get(0) + ".git") != null) { + referenceExpanded = subEntry.get(0); + break; + } + } + } + if (referenceExpanded == null) { + referenceExpanded = subEntries.iterator().next().get(0); + } + LOGGER.log( + Level.FINE, + "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + + "' from subEntries"); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } + } else { + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); + // If there is no hit, the non-fallback mode suggests a new + // directory name to host the submodule (same rules as for + // the refrepo forks' co-hosting friendly basename search), + // and the fallback mode would return the main directory. + int sep = url.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = url; + } else { + needleBasename = url.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll("\\.git$", ""); + + if (reference.endsWith("/${GIT_SUBMODULES}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", Matcher.quoteReplacement(needleBasename)); + } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } + } + } + } + + if (referenceExpanded != null) { + reference = referenceExpanded; + referencePath = new File(reference); + } + + LOGGER.log( + Level.INFO, + "After resolving the parameterized Git reference repository, " + "decided to use '" + reference + + "' directory for URL '" + url + "'"); + } // if referencePath is the replaceable token and not existing directory + + if (!referencePath.exists() && !reference.endsWith(".git")) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = new File(reference); + } + + // Note that the referenced path may exist or not exist, in the + // latter case it is up to the caller to decide on course of action. + // Maybe create this dir to begin a reference repo (given the options)? + return referencePath; + } + /** {@inheritDoc} */ @Override @Deprecated diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index 1b5e486838..5dbbca29be 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -345,6 +345,18 @@ public String getRemoteUrl(String name) throws GitException, InterruptedExceptio return proxy.getRemoteUrl(name); } + /** {@inheritDoc} */ + @Override + public Map getRemoteUrls() throws GitException, InterruptedException { + return proxy.getRemoteUrls(); + } + + /** {@inheritDoc} */ + @Override + public Map getRemotePushUrls() throws GitException, InterruptedException { + return proxy.getRemotePushUrls(); + } + /** {@inheritDoc} */ @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { @@ -709,6 +721,11 @@ public GitClient subGit(String subdir) { return proxy.subGit(subdir); } + /** {@inheritDoc} */ + public GitClient newGit(String somedir) { + return proxy.newGit(somedir); + } + /** * hasGitModules. * diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b1a25c23ae..3bd01ae77a 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -45,6 +45,8 @@ @MethodSource("gitObjects") class GitClientCloneTest { + private static final Logger LOGGER = Logger.getLogger(GitClientCloneTest.class.getName()); + @RegisterExtension private final GitClientSampleRepoRule repo = new GitClientSampleRepoRule(); @@ -267,6 +269,255 @@ void test_clone_reference() throws Exception { is(true)); } + @Test + void test_clone_reference_parameterized_basename() throws Exception { + /* This test clones from a local mirror directory rather than a real + * network URL. Git accepts forward slashes in local paths on every OS + * (including Windows), so spell the clone URL with forward slashes: + * this same string is also the "url" argument that + * findParameterizedReferenceRepository() extracts a basename from via + * url.lastIndexOf("/") - a backslash-only Windows path has no '/' to + * split on at all (its "basename" would be the whole path, which is + * not a legal single directory-name component). Forward slashes let + * the basename resolve to a portable value ("clone") on every OS, + * matching how a real repository URL behaves. */ + String wsMirrorUrl = workspace.localMirror().replace(File.separatorChar, '/'); + String wsMirrorBasename = + wsMirrorUrl.substring(wsMirrorUrl.lastIndexOf('/') + 1).replaceAll("\\.git$", ""); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepoBasename.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepoBasename.git/" + wsMirrorBasename); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); + + testGitClient + .clone_() + .url(wsMirrorUrl) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_BASENAME}") + .execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + // Not using check_remote_url(): that helper compares against + // workspace.localMirror()'s OS-separator spelling, but this test + // deliberately clones using the forward-slash spelling above. + assertThat("Remote URL", testGitClient.getRemoteUrl("origin"), is(wsMirrorUrl)); + assertThat("Updated URL", workspace.launchCommand("git", "remote", "-v"), containsString(wsMirrorUrl)); + + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); + + assertThat( + "Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")), + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: ") + && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)), + is(true)); + + if (wsRefrepo != null) { + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure + } + + @Test + void test_clone_reference_parameterized_basename_fallback() throws Exception { + // TODO: Currently we do not make paths which would invalidate + // this test, but note the test above might do just that later. + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}") + .execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat( + "Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), + is(true)); + // With fallback mode, and nonexistent parameterized reference + // repository, and a usable repository in the common path (what + // remains if the parameterizing suffix is just discarded), this + // common path should be used. So it should overall behave same + // as the non-parameterized test_clone_reference_basename() above. + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } + + @Test + void test_clone_reference_parameterized_sha256() throws Exception { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); + + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256}") + .execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); + + assertThat( + "Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")), + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: ") + && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)), + is(true)); + + if (wsRefrepo != null) { + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure + } + + @Test + void test_clone_reference_parameterized_sha256_fallback() throws Exception { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); + + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}") + .execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); + + // Note: we do not expect the closing single quote after wsRefrepoBase + // because other tests might pollute our test area, and SHA dir is there + assertThat( + "Reference repo name-parsing logged in: " + messages + "\n...and replaced with: '" + wsRefrepoBase, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), + is(true)); + + // Barring filesystem errors, if we have the "custom" refrepo + // we expect it to be used (fallback mode is not triggered) + if (wsRefrepo != null) { + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test @@ -451,13 +702,53 @@ private void assertNoObjectsInRepository() { } } + // Most tests use this method, expecting a non-bare repo private void assertAlternateFilePointsToLocalMirror() throws Exception { + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws Exception { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws Exception { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) + throws Exception { final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } - assertThat(new File(testGitDir, alternates), is(anExistingFile())); - final String expectedContent = workspace.localMirror().replace("\\", "/") + "/objects"; - final String actualContent = Files.readString(testGitDir.toPath().resolve(alternates), StandardCharsets.UTF_8); + // Similar for bare repos, without ".git/" dir + private void assertAlternateFilePointsToLocalBareMirror() throws Exception { + assertAlternateFilePointsToLocalBareMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws Exception { + assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws Exception { + assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws Exception { + final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } + + private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) + throws Exception { + assertThat( + "Did not find '" + alternates + "' under '" + _testGitDir + "'", + new File(_testGitDir, alternates), + is(anExistingFile())); + final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; + final String actualContent = + Files.readString(new File(_testGitDir).toPath().resolve(alternates), StandardCharsets.UTF_8); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java index 50942159c0..029c762bc7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java @@ -1016,6 +1016,52 @@ void testAddRemoteUrl() throws Exception { assertEquals(upstreamRepoURL, gitClient.getRemoteUrl("upstream")); } + @Test + void testGetRemoteUrls() throws Exception { + Map remoteUrls = gitClient.getRemoteUrls(); + assertNotNull(remoteUrls, "getRemoteUrls() returned null"); + assertThat(remoteUrls, hasEntry(srcRepoDir.getAbsolutePath(), "origin")); + } + + @Test + void testGetRemoteUrlsMultipleRemotes() throws Exception { + gitClient.addRemoteUrl("upstream", upstreamRepoURL); + Map remoteUrls = gitClient.getRemoteUrls(); + assertNotNull(remoteUrls, "getRemoteUrls() returned null"); + assertThat(remoteUrls, hasEntry(srcRepoDir.getAbsolutePath(), "origin")); + assertThat(remoteUrls.values(), hasItem("upstream")); + } + + @Test + void testGetRemotePushUrls_noPushUrl() throws Exception { + // No pushurl configured — implementations return null or empty map + Map pushUrls = gitClient.getRemotePushUrls(); + assertTrue( + pushUrls == null || pushUrls.isEmpty(), + "Expected null or empty map when no pushurl is configured, got: " + pushUrls); + } + + @Test + void testGetRemotePushUrls_withPushUrl() throws Exception { + String pushUrl = "git@github.com:example/repo.git"; + CliGitCommand gitCmd = new CliGitCommand(gitClient); + gitCmd.run("config", "remote.origin.pushurl", pushUrl); + Map pushUrls = gitClient.getRemotePushUrls(); + assertNotNull(pushUrls, "getRemotePushUrls() returned null after configuring pushurl"); + assertThat("origin remote should appear in push URL map", pushUrls.values(), hasItem("origin")); + } + + @Test + void testNewGit() throws Exception { + File otherDir = newFolder(tempFolder, "junit-newgit-" + System.nanoTime()); + GitClient otherClient = gitClient.newGit(otherDir.getAbsolutePath()); + assertNotNull(otherClient, "newGit() returned null"); + assertNotSame(gitClient, otherClient, "newGit() returned the same instance"); + // Verify the new client is a usable GitClient, not just a non-null reference + otherClient.init(); + assertTrue(new File(otherDir, ".git").isDirectory(), "newGit() client did not init a git repo"); + } + @Test void testAutocreateFailsOnMultipleMatchingOrigins() throws Exception { File repoRootTemp = newFolder(tempFolder, "junit-" + System.nanoTime()); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java new file mode 100644 index 0000000000..b1350dac10 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java @@ -0,0 +1,274 @@ +package org.jenkinsci.plugins.gitclient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import hudson.EnvVars; +import hudson.model.TaskListener; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Unit tests for the static/utility methods of {@link LegacyCompatibleGitAPIImpl}: + * {@code getObjectsFile()}, {@code isParameterizedReferenceRepository()}, + * and {@code normalizeGitUrl()}. + */ +class LegacyCompatibleGitAPIImplStaticTest { + + @TempDir + private File tempDir; + + private File repoDir; + + @BeforeEach + void setUp() throws Exception { + repoDir = newFolder(tempDir, "repo"); + } + + // ----------------------------------------------------------------------- + // getObjectsFile() + // ----------------------------------------------------------------------- + + @Test + void getObjectsFile_nullString_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile((String) null)); + } + + @Test + void getObjectsFile_emptyString_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile("")); + } + + @Test + void getObjectsFile_nullFile_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile((File) null)); + } + + @Test + void getObjectsFile_nonExistentPath_returnsNull() { + File missing = new File(tempDir, "does-not-exist"); + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile(missing)); + } + + @Test + void getObjectsFile_regularDirectory_returnsNull() { + // A plain directory without any git metadata should return null + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir)); + } + + @Test + void getObjectsFile_normalWorkspace_returnsObjectsDir() throws Exception { + // A standard (non-bare) git repo has .git/objects/ + Git.with(TaskListener.NULL, new EnvVars()) + .in(repoDir) + .using("git") + .getClient() + .init(); + File expected = new File(repoDir, ".git/objects"); + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for a normal workspace"); + assertThat(result.getAbsolutePath(), is(expected.getAbsolutePath())); + assertTrue(result.isDirectory(), "objects dir should be a directory"); + } + + @Test + void getObjectsFile_bareRepository_returnsObjectsDir() throws Exception { + // A bare repo has objects/ directly in the repo root + Git.with(TaskListener.NULL, new EnvVars()) + .in(repoDir) + .using("git") + .getClient() + .init_() + .workspace(repoDir.getAbsolutePath()) + .bare(true) + .execute(); + File expected = new File(repoDir, "objects"); + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for a bare repo"); + assertThat(result.getAbsolutePath(), is(expected.getAbsolutePath())); + assertTrue(result.isDirectory(), "objects dir should be a directory"); + } + + @Test + void getObjectsFile_submoduleGitPointerFile_returnsObjectsDir() throws Exception { + // Submodule workspaces have a .git FILE (not dir) containing "gitdir: " + // The real git metadata lives at the path referenced by that file. + File realGitDir = newFolder(tempDir, "real-gitdir"); + File realObjects = new File(realGitDir, "objects"); + assertTrue(realObjects.mkdir(), "Failed to create fake objects dir"); + + // Write a .git pointer file in repoDir + File gitPointer = new File(repoDir, ".git"); + String relativeGitdirPath = "../real-gitdir"; + Files.writeString(gitPointer.toPath(), "gitdir: " + relativeGitdirPath + "\n", StandardCharsets.UTF_8); + + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for submodule pointer workspace"); + assertTrue(result.isDirectory(), "Expected objects dir to be a real directory"); + // Canonical path resolves the ".." in the relative gitdir reference + assertThat(result.getCanonicalPath(), is(realObjects.getCanonicalPath())); + } + + // ----------------------------------------------------------------------- + // isParameterizedReferenceRepository() + // ----------------------------------------------------------------------- + + @Test + void isParameterizedReferenceRepository_null_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository((String) null)); + } + + @Test + void isParameterizedReferenceRepository_emptyString_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("")); + } + + @Test + void isParameterizedReferenceRepository_plainPath_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/repos")); + } + + @Test + void isParameterizedReferenceRepository_sha256Suffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_URL_SHA256}")); + } + + @Test + void isParameterizedReferenceRepository_sha256FallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_SHA256_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_basenameSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_URL_BASENAME}")); + } + + @Test + void isParameterizedReferenceRepository_basenameFallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_BASENAME_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_submodulesSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_SUBMODULES}")); + } + + @Test + void isParameterizedReferenceRepository_submodulesFallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_SUBMODULES_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_tokenInMiddle_returnsFalse() { + // Token must be at the END of the path; mid-path token is not recognized + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_SHA256}/extra")); + } + + @Test + void isParameterizedReferenceRepository_fileOverload_delegatesToStringVersion() { + // The File overload delegates to the String overload via file.getPath(). + // On Windows, getPath() uses backslashes while the checks use forward slashes, + // so we only assert the behaviors that are platform-independent: + // null File -> false, and non-parameterized File -> false. + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository((File) null)); + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(new File("/var/cache/git/repos"))); + } + + // ----------------------------------------------------------------------- + // normalizeGitUrl() + // ----------------------------------------------------------------------- + + @Test + void normalizeGitUrl_trailingSlashes_areStripped() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo///", false); + assertThat(result, not(endsWith("/"))); + } + + @Test + void normalizeGitUrl_dotGitSuffix_isStripped() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git", false); + assertThat(result, not(endsWith(".git"))); + } + + @Test + void normalizeGitUrl_mixedCaseDotGit_lowercasedButNotStripped() { + // The regex strips only exact lowercase ".git"; ".Git" is lowercased to ".git" but stays + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.Git", false); + assertThat(result, endsWith(".git")); + } + + @Test + void normalizeGitUrl_httpsUrl_isLowercased() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("HTTPS://GitHub.com/Example/Repo.git", false); + assertThat(result, startsWith("https://")); + assertThat(result, containsString("github.com")); + } + + @Test + void normalizeGitUrl_absoluteLocalPath_getsFileScheme() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo", false); + assertThat(result, startsWith("file://")); + } + + @Test + void normalizeGitUrl_absoluteLocalPath_dotGitStripped() { + String withGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo.git", false); + String withoutGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo", false); + assertThat(withGit, is(withoutGit)); + } + + @Test + void normalizeGitUrl_relativeLocalPath_getsFileScheme() { + // A path starting with "./" is treated as a local relative path + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("./some/local/repo", false); + assertThat(result, startsWith("file://")); + } + + @Test + void normalizeGitUrl_bareHostname_getsSshScheme() { + // A string without schema, not starting with / or ., is treated as an ssh host + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("git@github.com:example/repo.git", false); + assertThat(result, startsWith("ssh://")); + } + + @Test + void normalizeGitUrl_sameUrlDifferentSuffixes_normalizeToSame() { + String base = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo", false); + String withGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git", false); + String withSlash = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo/", false); + String withBoth = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git/", false); + assertThat(withGit, is(base)); + assertThat(withSlash, is(base)); + assertThat(withBoth, is(base)); + } + + // ----------------------------------------------------------------------- + // Helper + // ----------------------------------------------------------------------- + + private static File newFolder(File root, String... subDirs) throws IOException { + String subFolder = String.join("/", subDirs); + File result = new File(root, subFolder); + if (!result.mkdirs()) { + throw new IOException("Couldn't create folders " + result); + } + return result; + } +} diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java index 04a8e380da..6f20c76d7d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java @@ -1,5 +1,9 @@ package org.jenkinsci.plugins.gitclient; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.*; import hudson.model.TaskListener; @@ -12,9 +16,13 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.AbstractMap; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.RemoteConfig; @@ -249,6 +257,238 @@ void testExtractBranchNameFromBranchSpec() { assertEquals("refs/tags/mytag", git.extractBranchNameFromBranchSpec("refs/tags/mytag")); } + // ----------------------------------------------------------------------- + // getSubmodulesUrls() and findParameterizedReferenceRepository() tests [JENKINS-64383] + // NOTE: The name "getSubmodulesUrls" is a bit misleading here, so far. + // It's named for the use case (a fanout reference repo that *might* contain + // submodule-like repos), but the implementation currently looks for + // "any subdir with a git objects dir", not for registered git submodules. + // Code has a "TODO" that is not implemented yet: + // If current repo is NOT bare, also check git submodules + // registered in .gitmodules for a faster possible answer + // ----------------------------------------------------------------------- + + /** + * Creates a subdirectory inside {@code parent}, initializes a git repo + * there, and registers a remote "origin" pointing to {@code remoteUrl}. + */ + private File createSubdirWithRemote(File parent, String dirName, String remoteUrl) throws Exception { + File subDir = newFolder(parent, dirName); + GitClient subGit = Git.with(listener, env).in(subDir).using(gitImpl).getClient(); + subGit.init(); + new CliGitCommand(subGit).run("remote", "add", "origin", remoteUrl); + return subDir; + } + + @Test + void testGetSubmodulesUrls_noNeedle_returnsAllSubdirs() throws Exception { + File base = newFolder(tempFolder, "refrepo-noneedle"); + File repoAlpha = createSubdirWithRemote(base, "alpha-repo", "https://github.com/example/alpha-project.git"); + File repoBeta = createSubdirWithRemote(base, "beta-repo", "https://github.com/example/beta-project.git"); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath()); + + assertFalse(result.getKey(), "No needle: flag should be false"); + LinkedHashSet> entries = result.getValue(); + assertFalse(entries.isEmpty(), "Should find at least one subdir"); + Set foundDirs = + entries.stream().map(e -> new File(e.get(0)).getAbsolutePath()).collect(Collectors.toSet()); + assertThat("alpha-repo should appear in results", foundDirs, hasItem(repoAlpha.getAbsolutePath())); + assertThat("beta-repo should appear in results", foundDirs, hasItem(repoBeta.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_exactNeedleMatch_promotedToFirst() throws Exception { + // "target-basename" dir is named to match the needle's URL basename, so it is + // probed in the prefix-search phase (before any listFiles() walk). It holds a + // *different* URL -> no exact match -> accumulated into result first. + // "aaa-exact" holds the exact needle URL and is found in the general walk that + // follows. The exact-match return promotes it to result_best[0] with all + // previously accumulated entries after it, giving at least 2 entries total. + String needleUrl = "https://github.com/example/target-basename.git"; + File base = newFolder(tempFolder, "refrepo-exactmatch"); + createSubdirWithRemote(base, "target-basename", "https://github.com/other-org/not-the-target.git"); + File repoExact = createSubdirWithRemote(base, "aaa-exact", needleUrl); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Exact URL match: flag should be true"); + LinkedHashSet> entries = result.getValue(); + assertThat( + "Exact-match entry + previously accumulated entry = at least 2", + entries.size(), + greaterThanOrEqualTo(2)); + String firstDir = entries.iterator().next().get(0); + assertThat( + "Exact-match repo is promoted to first position", + new File(firstDir).getAbsolutePath(), + is(repoExact.getAbsolutePath())); + } + + @Test + void testFindParameterizedReferenceRepository_submodules_exactMatchPickedAsResult() throws Exception { + // Same two-repo setup: prefix-search accumulates "target-basename" (non-exact), + // then the general walk finds "aaa-exact" (exact). The consuming code uses the + // first entry in the result set, which is the exact match. + String needleUrl = "https://github.com/example/target-basename.git"; + File base = newFolder(tempFolder, "refrepo-find-exact"); + createSubdirWithRemote(base, "target-basename", "https://github.com/other-org/not-the-target.git"); + File repoExact = createSubdirWithRemote(base, "aaa-exact", needleUrl); + String reference = base.getAbsolutePath() + "/${GIT_SUBMODULES}"; + + File result = git.findParameterizedReferenceRepository(reference, needleUrl); + + assertNotNull(result, "Should return a result for an exact URL match"); + assertThat( + "Exact-match subdir should be returned", result.getCanonicalPath(), is(repoExact.getCanonicalPath())); + } + + @Test + void testFindParameterizedReferenceRepository_submodules_firstEntryUsedForNonExact() throws Exception { + // "cool-project" dir matches the needle's URL basename and is probed first in + // the prefix-search phase -> ends up first in the ordered suggestion set. + // "another-fork" is found in the general walk with the same URL basename. + // With no exact match, findParameterizedReferenceRepository iterates the set + // and returns the first entry that is an existing git repo -> "cool-project". + String needleUrl = "https://github.com/example/cool-project.git"; + File base = newFolder(tempFolder, "refrepo-find-nonexact"); + File repoFirst = createSubdirWithRemote(base, "cool-project", "https://github.com/fork-a/cool-project.git"); + createSubdirWithRemote(base, "another-fork", "https://github.com/fork-b/cool-project.git"); + String reference = base.getAbsolutePath() + "/${GIT_SUBMODULES}"; + + File result = git.findParameterizedReferenceRepository(reference, needleUrl); + + assertNotNull(result, "Should return a result when non-exact URL-basename matches exist"); + assertThat( + "First entry from the ordered suggestion set should be returned", + result.getCanonicalPath(), + is(repoFirst.getCanonicalPath())); + } + + @Test + void testGetSubmodulesUrls_mapFileHit_skipsDirectoryWalk() throws Exception { + String needleUrl = "ssh://git@example.com/org/mapped-project.git"; + File base = newFolder(tempFolder, "refrepo-map-hit"); + File repoMapped = createSubdirWithRemote(base, "mapped-project.git", needleUrl); + // A decoy that would be found first by the ordinary directory walk if + // the mapping file were not consulted, so a wrong hit here would fail + // the assertion on the returned directory below. + createSubdirWithRemote(base, "aaa-decoy", needleUrl); + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-1717010012\t" + needleUrl + "\tmapped-project.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Map file exact match: flag should be true"); + LinkedHashSet> entries = result.getValue(); + assertEquals(1, entries.size(), "Map-file hit should short-circuit with exactly one entry"); + List hit = entries.iterator().next(); + assertThat( + "Map-file hit should point at the mapped dir", + new File(hit.get(0)).getAbsolutePath(), + is(repoMapped.getAbsolutePath())); + assertEquals( + "origin", + hit.get(3), + "remoteName should come from verifying the actual dir's remotes, not the map file's first column"); + } + + @Test + void testGetSubmodulesUrls_mapFileStaleEntry_fallsBackToDirectoryWalk() throws Exception { + String needleUrl = "ssh://git@example.com/org/still-findable.git"; + File base = newFolder(tempFolder, "refrepo-map-stale"); + File repoReal = createSubdirWithRemote(base, "still-findable.git", needleUrl); + // Map file points at a subdirectory that does not actually exist (as if + // the cache had been pruned/renamed after the map was last saved). + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-0000000000\t" + needleUrl + "\tgone-missing.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory-walk fallback should still find the real dir"); + LinkedHashSet> entries = result.getValue(); + assertThat( + "Fallback should find the real dir, not the stale map entry", + new File(entries.iterator().next().get(0)).getAbsolutePath(), + is(repoReal.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_mapFileEntryWithoutMatchingRemote_fallsBackToDirectoryWalk() throws Exception { + // The mapped directory exists and is a real git repo, but its remote + // was changed (or the dir was repurposed) since the map was saved, so + // it no longer actually has the needle URL configured - the map's + // dirname-existence check alone would wrongly trust this hit. + String needleUrl = "ssh://git@example.com/org/moved-away.git"; + File base = newFolder(tempFolder, "refrepo-map-wrong-remote"); + createSubdirWithRemote(base, "moved-away.git", "ssh://git@example.com/org/renamed-elsewhere.git"); + File repoReal = createSubdirWithRemote(base, "aaa-actual-hit", needleUrl); + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-1234567890\t" + needleUrl + "\tmoved-away.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory-walk fallback should still find the real dir"); + LinkedHashSet> entries = result.getValue(); + assertThat( + "Fallback should find the dir that actually has the needle remote, not the mismatched map entry", + new File(entries.iterator().next().get(0)).getAbsolutePath(), + is(repoReal.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_noMapFile_behavesAsBefore() throws Exception { + String needleUrl = "ssh://git@example.com/org/no-map-here.git"; + File base = newFolder(tempFolder, "refrepo-no-map"); + File repoReal = createSubdirWithRemote(base, "no-map-here.git", needleUrl); + assertFalse( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).exists(), + "Precondition: no map file present"); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory walk should still find the exact match without a map file"); + assertThat( + new File(result.getValue().iterator().next().get(0)).getAbsolutePath(), is(repoReal.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_configWithIncludeDirective_stillFindsRemoteViaFallback() throws Exception { + // getRemoteUrlsFast() refuses to parse a "config" file with an + // include/includeIf directive directly (remotes could be defined in + // the included file, which it does not resolve), and falls back to + // a full GitClient lookup instead. Confirm that fallback still works + // and still finds the correct match. + String needleUrl = "ssh://git@example.com/org/has-include.git"; + File base = newFolder(tempFolder, "refrepo-config-include"); + File repoDir = createSubdirWithRemote(base, "has-include.git", needleUrl); + File configFile = new File(repoDir, ".git/config"); + Files.writeString( + configFile.toPath(), + "[include]\n\tpath = /nonexistent-included-file-xyz\n", + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Fallback via GitClient should still find the exact match"); + assertThat( + new File(result.getValue().iterator().next().get(0)).getAbsolutePath(), is(repoDir.getAbsolutePath())); + } + private static File newFolder(File root, String... subDirs) throws Exception { String subFolder = String.join("/", subDirs); File result = new File(root, subFolder); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index f8a95abb86..3bee8d0029 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -16,12 +16,17 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.lib.ObjectId; class WorkspaceWithRepo { + private static final Logger LOGGER = Logger.getLogger(WorkspaceWithRepo.class.getName()); + private GitClient gitClient; private File gitFileDir; private CliGitCommand cliGitCommand; @@ -85,12 +90,22 @@ private boolean isShallow() { * @throws InterruptedException when exception is interrupted */ String localMirror() throws Exception { + return localMirror("clone.git"); + } + + String localMirror(String cloneDirName) throws Exception { File base = new File(".").getAbsoluteFile(); + LOGGER.log(Level.FINE, "Beginning to search for cloneDirName=''{0}'' from {1}", new Object[] { + cloneDirName, base.getPath() + }); for (File f = base; f != null; f = f.getParentFile()) { + LOGGER.log(Level.FINE, "Looking for ''target'' in {0}", f.getPath()); File targetDir = new File(f, "target"); if (targetDir.exists()) { - String cloneDirName = "clone.git"; File clone = new File(targetDir, cloneDirName); + LOGGER.log(Level.FINE, "Looking for cloneDirName {0} in {1}", new Object[] { + cloneDirName, targetDir.getPath() + }); if (!clone.exists()) { /* Clone to a temporary directory then move the * temporary directory to the final destination @@ -102,6 +117,7 @@ String localMirror() throws Exception { */ Path tempClonePath = Files.createTempDirectory(targetDir.toPath(), "clone-"); String destination = tempClonePath.toFile().getAbsolutePath(); + LOGGER.log(Level.FINE, "tempClonePath={0} => (FQPN){1}", new Object[] {tempClonePath, destination}); if (isShallow()) { cliGitCommand.run("clone", "--mirror", repoURL, destination); } else { @@ -109,6 +125,7 @@ String localMirror() throws Exception { "clone", "--reference", f.getCanonicalPath(), "--mirror", repoURL, destination); } if (!clone.exists()) { // Still a race condition, but a narrow race handled by Files.move() + LOGGER.log(Level.FINE, "moving tempClonePath to cloneDirName={0}", cloneDirName); renameAndDeleteDir(tempClonePath, cloneDirName); } else { /* @@ -133,10 +150,19 @@ String localMirror() throws Exception { * deleteRecursive() will discard a clone that * 'lost the race'. */ + LOGGER.log( + Level.FINE, + "removing extra tempClonePath, we already (race?) have cloneDirName={0}", + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } + } else { + LOGGER.log( + Level.FINE, "FOUND cloneDirName {0} in {1}", new Object[] {cloneDirName, targetDir.getPath() + }); } - return clone.getPath(); + // Strip away the "/./" in "...git-client-plugin/./target/..." + return Paths.get(clone.getPath()).normalize().toString(); } } throw new IllegalStateException();