From 8f5ec330c99379f186723c70da1fdac07f985fa5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 7 Dec 2020 22:24:59 +0100 Subject: [PATCH 001/155] Introduce support for reference repository paths ending with /${GIT_URL} to replace by url => funny dir subtree in filesystem --- .../plugins/gitclient/CliGitAPIImpl.java | 33 +++++++++++++++++++ .../plugins/gitclient/JGitAPIImpl.java | 21 ++++++++++++ .../plugins/gitclient/GitClientCloneTest.java | 15 +++++++++ 3 files changed, 69 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 521958e87d..ab7904b88d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -762,6 +762,39 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // See also JGitAPIImpl.java and keep the two blocks in + // sync (TODO: refactor into a method both would use?) + // For mass-configured jobs, like Organization Folders, + // allow to support parameterized paths to many refrepos. + // End-users can set up webs of symlinks to same repos + // known by different URLs (and/or including their forks + // also cached in same index). Luckily all URL chars are + // valid parts of path name... in Unix... Maybe parse or + // escape chars for URLs=>paths with Windows in mind? + // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions + // Further ideas: beside "GIT_URL" other meta variables + // can be introduced, e.g. to escape non-ascii chars for + // portability? Support SHA or MD5 hashes of URLs as + // pathnames? + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = new File(reference); + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = new File(reference); + } + // Note: both these logs are needed, they are used in selftest + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + } else { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + } + } + } + if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a1ffa8acb1..4c34b9a386 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1406,6 +1406,27 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // See comments in CliGitAPIImpl.java for details + // Keep the two blocks in sync (refactor into method?) + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = new File(reference); + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = new File(reference); + } + // Note: both these logs are needed, they are used in selftest + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + } else { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + } + } + } + if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index c44a06689e..d273d46d13 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -238,6 +238,21 @@ public void test_clone_reference() throws Exception, IOException, InterruptedExc assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); } + @Test + public void test_clone_reference_parameterized() throws Exception, IOException, InterruptedException { + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL}").execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path replaced with: "), is(true)); + // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + //assertAlternateFilePointsToLocalMirror(); + //assertBranchesExist(testGitClient.getBranches(), "master"); + //assertNoObjectsInRepository(); + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From 022bbc6d4f1258b7edd6573fb56be3f5601aa9a4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 16:58:37 +0100 Subject: [PATCH 002/155] Refactor: move findParameterizedReferenceRepository() into LegacyCompatibleGitAPIImpl.java so its logic (expected to grow in complexity) can be shared by both JGitAPIImpl.java and CliGitAPIImpl.java --- .../plugins/gitclient/CliGitAPIImpl.java | 88 ++++++++----------- .../plugins/gitclient/JGitAPIImpl.java | 75 ++++++++-------- .../gitclient/LegacyCompatibleGitAPIImpl.java | 50 +++++++++++ .../plugins/gitclient/GitClientCloneTest.java | 5 +- 4 files changed, 128 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index ab7904b88d..18d8b56c93 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -761,64 +761,50 @@ public void execute() throws GitException, InterruptedException { } if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - if (reference.endsWith("/${GIT_URL}")) { - // See also JGitAPIImpl.java and keep the two blocks in - // sync (TODO: refactor into a method both would use?) - // For mass-configured jobs, like Organization Folders, - // allow to support parameterized paths to many refrepos. - // End-users can set up webs of symlinks to same repos - // known by different URLs (and/or including their forks - // also cached in same index). Luckily all URL chars are - // valid parts of path name... in Unix... Maybe parse or - // escape chars for URLs=>paths with Windows in mind? - // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions - // Further ideas: beside "GIT_URL" other meta variables - // can be introduced, e.g. to escape non-ascii chars for - // portability? Support SHA or MD5 hashes of URLs as - // pathnames? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); - referencePath = new File(reference); - if (!referencePath.exists()) { - // Normalize the URLs with or without .git suffix to - // be served by same dir with the refrepo contents - reference += ".git"; - referencePath = new File(reference); - } + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + } else { + if (!referencePath.getName().equals(reference)) { // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path "; + msg += "'" + reference + "'"; + msg += " replaced with: "; + msg += "'" + referencePath.getName() + "'"; if (referencePath.exists()) { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + listener.getLogger().println("[WARNING] " + msg); } else { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + listener.getLogger().println("[WARNING] " + msg + " does not exist"); } + reference = referencePath.getName(); } - } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File alternates = new File(workspace, ".git/objects/info/alternates"); - try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - w.print(absoluteReference); - } catch (UnsupportedEncodingException ex) { - listener.error("Default character set is an unsupported encoding"); - } catch (FileNotFoundException e) { - listener.error("Failed to setup reference"); + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + } + if (!objectsPath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + else { + // Go behind git's back to write a meta file in new workspace + File alternates = new File(workspace, ".git/objects/info/alternates"); + try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { + String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); + // git implementations on windows also use + w.print(absoluteReference); + } catch (UnsupportedEncodingException ex) { + listener.error("Default character set is an unsupported encoding"); + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); + } } } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 4c34b9a386..323b386518 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1405,53 +1405,52 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - if (reference.endsWith("/${GIT_URL}")) { - // See comments in CliGitAPIImpl.java for details - // Keep the two blocks in sync (refactor into method?) - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); - referencePath = new File(reference); - if (!referencePath.exists()) { - // Normalize the URLs with or without .git suffix to - // be served by same dir with the refrepo contents - reference += ".git"; - referencePath = new File(reference); - } + // Note: keep in sync with similar logic in CliGitAPIImpl.java + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + } else { + if (!referencePath.getName().equals(reference)) { // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path "; + msg += "'" + reference + "'"; + msg += " replaced with: "; + msg += "'" + referencePath.getName() + "'"; if (referencePath.exists()) { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + listener.getLogger().println("[WARNING] " + msg); } else { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + listener.getLogger().println("[WARNING] " + msg + " does not exist"); } + reference = referencePath.getName(); } - } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - try { - File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + } + if (!objectsPath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + else { + // Go behind git's back to write a meta file in new workspace + try { + File alternates = new File(workspace, ".git/objects/info/alternates"); + String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use - w.print(absoluteReference); + try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + // git implementations on windows also use + w.print(absoluteReference); + } + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); } - } catch (FileNotFoundException e) { - listener.error("Failed to setup reference"); } } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 24ec1b7289..8654e4f6a8 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -158,6 +158,56 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** Yield the File object for the reference repository local filesystem + * pathname. Note that the provided string may be suffixed with expandable + * tokens which allow to store a filesystem structure of multiple small + * reference repositories instead of a big combined repository, while + * providing a single inheritable configuration string value. Callers + * can check whether the original path was used or mangled into another + * by comparing their "reference" with returned object's File.getName(). + * + * At some point this plugin might also maintain that filesystem structure. + */ + public File findParameterizedReferenceRepository(String reference, String url) { + if (reference == null || reference.isEmpty()) { + return null; + } + + File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // For mass-configured jobs, like Organization Folders, + // allow to support parameterized paths to many refrepos. + // End-users can set up webs of symlinks to same repos + // known by different URLs (and/or including their forks + // also cached in same index). Luckily all URL chars are + // valid parts of path name... in Unix... Maybe parse or + // escape chars for URLs=>paths with Windows in mind? + // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions + // Further ideas: beside "GIT_URL" other meta variables + // can be introduced, e.g. to escape non-ascii chars for + // portability? Support base64, SHA or MD5 hashes of URLs + // as pathnames? Normalize first (lowercase, .git, ...)? + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = null; // GC + referencePath = new File(reference); + } + } + + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = null; // GC + referencePath = new File(reference); + } + + // Note that the referenced path may exist or not exist, in the + // latter case it is up to the caller to decide on course of action. + // Maybe create this dir to begin a reference repo (given the options)? + return referencePath; + } + /** {@inheritDoc} */ @Deprecated public List revListBranch(String branchId) throws GitException, InterruptedException { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index d273d46d13..7fb6d08324 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -245,7 +245,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path replaced with: "), is(true)); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: "), + is(true)); // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); //assertAlternateFilePointsToLocalMirror(); From 6210adbb38e47fa69666113e8daec65fcca170d5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 17:52:03 +0100 Subject: [PATCH 003/155] Add isParameterizedReferenceRepository() and support for parameterized ref-repos in submodule checkouts (only CliGitAPIImpl.java has it) --- .../plugins/gitclient/CliGitAPIImpl.java | 37 +++++++++++++++---- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 18d8b56c93..1743f98257 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1396,13 +1396,15 @@ public void execute() throws GitException, InterruptedException { } } if ((ref != null) && !ref.isEmpty()) { - File referencePath = new File(ref); - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else - args.add("--reference", ref); + if (!isParameterizedReferenceRepository(ref)) { + File referencePath = new File(ref); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); + else + args.add("--reference", ref); + } // else handled below in per-module loop } if (shallow) { if (depth == null) { @@ -1453,6 +1455,27 @@ else if (!referencePath.isDirectory()) throw new GitException("Invalid repository for " + sModuleName); } + if (isParameterizedReferenceRepository(ref)) { + File referencePath = findParameterizedReferenceRepository(ref, urIish); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); + } else { + String expRef = null; + if (referencePath.getName().equals(ref)) { + expRef = ref; + } else { + expRef = referencePath.getName(); + expRef += " (expanded from " + ref + ")"; + } + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); + else + args.add("--reference", referencePath.getName()); + } + } + // Find credentials for this URL StandardCredentials cred = credentials.get(urIish.toPrivateString()); if (parentCredentials) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8654e4f6a8..e956eeaa6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -158,6 +158,20 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** Handle magic strings in the reference pathname to sort out patterns + * classified as evaluated by parametrization, as handled below */ + public Boolean isParameterizedReferenceRepository(String reference) { + if (reference == null || reference.isEmpty()) { + return false; + } + + if (reference.endsWith("/${GIT_URL}")) { + return true; + } + + return false; + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small From 06d4859c5244c14e199e9f12d8ba350da14664c5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:01:32 +0100 Subject: [PATCH 004/155] Handle the possibility that the URL for parameterized checkout is null String --- .../org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 8 ++++++-- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 1743f98257..deb3c9b990 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1454,9 +1454,13 @@ else if (!referencePath.isDirectory()) listener.error("Invalid repository for " + sModuleName); throw new GitException("Invalid repository for " + sModuleName); } + String strURIish = null; + if (urIish != null) { + strURIish = urIish.toPrivateString(); + } if (isParameterizedReferenceRepository(ref)) { - File referencePath = findParameterizedReferenceRepository(ref, urIish); + File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); } else { @@ -1477,7 +1481,7 @@ else if (!referencePath.isDirectory()) } // Find credentials for this URL - StandardCredentials cred = credentials.get(urIish.toPrivateString()); + StandardCredentials cred = credentials.get(strURIish); if (parentCredentials) { String parentUrl = getRemoteUrl(getDefaultRemote()); URIish parentUri = null; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e956eeaa6b..5e143bb8b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -188,7 +188,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); - if (!referencePath.exists()) { + if (!referencePath.exists() && url != null && !url.isEmpty()) { if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. From b5a6a7b1bae0f9f5c5dfdc7efd61689f38f96a87 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:30:01 +0100 Subject: [PATCH 005/155] LegacyCompatibleGitAPIImpl.java: document params for new routines --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5e143bb8b2..b0dde55f6f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -159,7 +159,11 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, } /** Handle magic strings in the reference pathname to sort out patterns - * classified as evaluated by parametrization, as handled below */ + * classified as evaluated by parametrization, as handled below + * + * @params + * reference Pathname (maybe with magic suffix) to reference repo + */ public Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; @@ -181,6 +185,11 @@ public Boolean isParameterizedReferenceRepository(String reference) { * by comparing their "reference" with returned object's File.getName(). * * At some point this plugin might also maintain that filesystem structure. + * + * @params + * reference Pathname (maybe with magic suffix) to reference repo + * url URL of the repository being cloned, to help choose a + * suitable parameterized reference repo subdirectory. */ public File findParameterizedReferenceRepository(String reference, String url) { if (reference == null || reference.isEmpty()) { From f90bdca702d70e18f0f3643d57c488bfede6689c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:31:10 +0100 Subject: [PATCH 006/155] LegacyCompatibleGitAPIImpl.java: refactor urlNormalized generation in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0dde55f6f..f602259685 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -198,6 +198,26 @@ public File findParameterizedReferenceRepository(String reference, String url) { File referencePath = new File(reference); if (!referencePath.exists() && url != null && !url.isEmpty()) { + // Note: this normalization might crush several URLs into one, + // and as far as is known this would be the goal - people tend + // to use or omit .git suffix, or spell with varied case, while + // it means the same thing to known Git platforms. + // The actual reference repository directory may choose to have + // original URL strings added as remotes (in case some are case + // sensitive and different). + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + if (!url.contains("://") && + !url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + File urlPath = new File(url); + if (!urlPath.exists()) { + // Also not a subdirectory without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } + if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -211,7 +231,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); referencePath = null; // GC referencePath = new File(reference); } From e09eff1eeeea78f96dd8bf7ce0d98d39468f688f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:34:14 +0100 Subject: [PATCH 007/155] LegacyCompatibleGitAPIImpl.java: introduce support for .../${GIT_URL_SHA256} reference repos --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f602259685..c9721a734c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1,6 +1,7 @@ package org.jenkinsci.plugins.gitclient; import static java.util.Arrays.copyOfRange; +import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; import hudson.model.TaskListener; import hudson.plugins.git.GitException; @@ -173,6 +174,10 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_SHA256}")) { + return true; + } + return false; } @@ -234,6 +239,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); referencePath = null; // GC referencePath = new File(reference); + } else if (reference.endsWith("/${GIT_URL_SHA256}")) { + // This may be the more portable solution with regard to filesystems + reference = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referencePath = null; // GC + referencePath = new File(reference); } } From 17010ca0849ec4dfcd1703309122bc848593ea32 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 22:36:03 +0100 Subject: [PATCH 008/155] WorkspaceWithRepo.java (test): extend localMirror() with customizable name for the mirrored repo --- .../org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 4602ad9acd..26192748d7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -76,11 +76,14 @@ private void renameAndDeleteDir(Path srcDir, String destDirName) { * @throws InterruptedException when execption is interrupted */ String localMirror() throws IOException, InterruptedException { + return localMirror("clone.git"); + } + + String localMirror(String cloneDirName) throws IOException, InterruptedException { File base = new File(".").getAbsoluteFile(); for (File f = base; f != null; f = f.getParentFile()) { File targetDir = new File(f, "target"); if (targetDir.exists()) { - String cloneDirName = "clone.git"; File clone = new File(targetDir, cloneDirName); if (!clone.exists()) { /* Clone to a temporary directory then move the From 72a4a05dc5c873896f0b6d2496fb67a202f9f960 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 22:36:33 +0100 Subject: [PATCH 009/155] GitClientCloneTest.java: add test_clone_reference_parameterized_sha256() with real sanity checks --- .../plugins/gitclient/GitClientCloneTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 7fb6d08324..79718d159d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -6,6 +6,7 @@ import hudson.plugins.git.GitException; import hudson.remoting.VirtualChannel; import hudson.util.StreamTaskListener; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; @@ -256,6 +257,53 @@ public void test_clone_reference_parameterized() throws Exception, IOException, //assertNoObjectsInRepository(); } + @Test + public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ + String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call + File fRefrepoBase = new File("refrepo.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: ") && + (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + is(true)); + + if (wsRefrepo != null) { + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From c6c010fe3472159b20aad000e8a77add122e8e33 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 01:13:03 +0100 Subject: [PATCH 010/155] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): do not bother normalizing the URL if the string is not with supported suffix --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c9721a734c..b9398b4c6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -202,7 +202,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); - if (!referencePath.exists() && url != null && !url.isEmpty()) { + if (!referencePath.exists() && + isParameterizedReferenceRepository(reference) && + url != null && !url.isEmpty() + ) { // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while From bbedefd508038fe9e2a9f9c296887b9870479e6b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 01:29:07 +0100 Subject: [PATCH 011/155] Fix referencePath getName() => getPath() --- .../jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 12 ++++++------ .../org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index deb3c9b990..40fca347a8 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -765,18 +765,18 @@ public void execute() throws GitException, InterruptedException { if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); } else { - if (!referencePath.getName().equals(reference)) { + if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest String msg = "Parameterized reference path "; msg += "'" + reference + "'"; msg += " replaced with: "; - msg += "'" + referencePath.getName() + "'"; + msg += "'" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { listener.getLogger().println("[WARNING] " + msg + " does not exist"); } - reference = referencePath.getName(); + reference = referencePath.getPath(); } if (!referencePath.exists()) @@ -1465,10 +1465,10 @@ else if (!referencePath.isDirectory()) listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); } else { String expRef = null; - if (referencePath.getName().equals(ref)) { + if (referencePath.getPath().equals(ref)) { expRef = ref; } else { - expRef = referencePath.getName(); + expRef = referencePath.getPath(); expRef += " (expanded from " + ref + ")"; } if (!referencePath.exists()) @@ -1476,7 +1476,7 @@ else if (!referencePath.isDirectory()) else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); else - args.add("--reference", referencePath.getName()); + args.add("--reference", referencePath.getPath()); } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 323b386518..5a03460604 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1410,18 +1410,18 @@ public void execute() throws GitException, InterruptedException { if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); } else { - if (!referencePath.getName().equals(reference)) { + if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest String msg = "Parameterized reference path "; msg += "'" + reference + "'"; msg += " replaced with: "; - msg += "'" + referencePath.getName() + "'"; + msg += "'" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { listener.getLogger().println("[WARNING] " + msg + " does not exist"); } - reference = referencePath.getName(); + reference = referencePath.getPath(); } if (!referencePath.exists()) From 5306163d6c69233e5de53ac1b5044ef45351da1e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 03:00:58 +0100 Subject: [PATCH 012/155] Update tests for fRefrepoBase being under target/ dir from CWD of the test runner --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 79718d159d..ba8f25b1a6 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -265,8 +265,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ - // Start of the path to pass into `git clone` call - File fRefrepoBase = new File("refrepo.git").getAbsoluteFile(); + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo.git").getAbsoluteFile(); String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname String wsRefrepo = null; try { From bc7e19819657f1fbfc5c902bb274fc4916a1bbe0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 03:25:24 +0100 Subject: [PATCH 013/155] GitClientCloneTest: check if refrepo dir exists already --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index ba8f25b1a6..b9a89f629d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -268,16 +268,16 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // Start of the path to pass into `git clone` call; note that per // WorkspaceWithRepo.java the test workspaces are under target/ // where the executed test binaries live - File fRefrepoBase = new File("target/refrepo.git").getAbsoluteFile(); + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname String wsRefrepo = null; try { - if (fRefrepoBase.mkdirs()) { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { /* Note: per parser of magic suffix, use slash - not OS separator char * And be sure to use relative paths here (see * WorkspaceWithRepo.java::localMirror()): */ - wsRefrepo = workspace.localMirror("refrepo.git/" + wsMirrorHash); + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { wsRefrepo = null; From 3ff19df7fb0d2c4ff468a30c829782b6880eb993 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:06:43 +0100 Subject: [PATCH 014/155] WorkspaceWithRepo.java: localMirror(): normalize() the returned pathname string --- .../org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 26192748d7..effea085aa 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -16,6 +16,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -124,7 +125,8 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException Util.deleteRecursive(tempClonePath.toFile()); } } - return clone.getPath(); + // Strip away the "/./" in "...git-client-plugin/./target/..." + return Paths.get(clone.getPath()).normalize().toString(); } } throw new IllegalStateException(); From a509386b15596f91faa9d1f1c9a6803d162c1b9c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:12:59 +0100 Subject: [PATCH 015/155] GitClientCloneTest.java: extend API choices for assertAlternateFilePointsToLocal*Mirror() with custom paths and bare vs workspace repos --- .../plugins/gitclient/GitClientCloneTest.java | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b9a89f629d..d4a4545588 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -414,12 +414,51 @@ private void assertNoObjectsInRepository() { } + // Most tests use this method, expecting a non-bare repo private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } - assertThat(new File(testGitDir, alternates), is(anExistingFile())); - final String expectedContent = workspace.localMirror().replace("\\", "/") + "/objects"; - final String actualContent = FileUtils.readFileToString(new File(testGitDir, alternates), "UTF-8"); + // Similar for bare repos, without ".git/" dir + private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { + final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } + + private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) throws IOException, InterruptedException { + assertThat("Did not find '" + alternates + "' under '" + _testGitDir + "'", + new File(_testGitDir, alternates), is(anExistingFile())); + final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; + final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); From 603e3900e96a8875853a3cca926025b4f1d0ebc0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:51:16 +0100 Subject: [PATCH 016/155] GitClientCloneTest.java: fix test_clone_reference_parameterized_sha256() to find expected paths --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index d4a4545588..b0a986ebfb 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -269,7 +269,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // WorkspaceWithRepo.java the test workspaces are under target/ // where the executed test binaries live File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); - String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { @@ -300,7 +300,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce if (wsRefrepo != null) { assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); - assertAlternateFilePointsToLocalMirror(); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure From 66e23a1322ac7d241908d071f5d5416ca56fa70c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:45:47 +0100 Subject: [PATCH 017/155] LegacyCompatibleGitAPIImpl.java CliGitAPIImpl.java JGitAPIImpl.java: refactor getObjectPath(referencePath) to check on git dirs elsewhere later --- .../plugins/gitclient/CliGitAPIImpl.java | 9 +--- .../plugins/gitclient/JGitAPIImpl.java | 9 +--- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 +++++++++++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 40fca347a8..b8c011a08c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -784,13 +784,8 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) + File objectsPath = getObjectPath(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); else { // Go behind git's back to write a meta file in new workspace diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 5a03460604..a83981b144 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1429,13 +1429,8 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) + File objectsPath = getObjectPath(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); else { // Go behind git's back to write a meta file in new workspace diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b9398b4c6b..2c5ab04961 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -159,6 +159,53 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** For referenced directory check if it is a full or bare git repo + * and return the File object for its "objects" sub-directory. + * If there is nothing to find, or inputs are bad, returns null. + * The idea is that checking for null allows to rule out non-git + * paths, while a not-null return value is instantly usable by + * some code which plays with git under its hood. + */ + public File getObjectPath(String reference) { + if (reference == null || reference.isEmpty()) { + return null; + } + return getObjectPath(new File(reference)); + } + + public File getObjectPath(File referencePath) { + if (referencePath == null) { + return referencePath; + } + + if (!referencePath.exists()) + return null; + + if (!referencePath.isDirectory()) + return null; + + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (objectsPath == null) { + return objectsPath; // Some Java error, could not make object from the paths involved + } + + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + if (objectsPath == null) { + return objectsPath; // Some Java error, could not make object from the paths involved + } + } + + if (!objectsPath.isDirectory()) + return null; + + // If we get here, we have a non-null File referencing a + // "(.git/)objects" subdir under original referencePath + return objectsPath; + } + /** Handle magic strings in the reference pathname to sort out patterns * classified as evaluated by parametrization, as handled below * From 5af44004edcdaa52e84a823676ac5c36ad6f816a Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:46:42 +0100 Subject: [PATCH 018/155] LegacyCompatibleGitAPIImpl.java: add variants of findParameterizedReferenceRepository() and isParameterizedReferenceRepository() taking a File reference (not only a String) object --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2c5ab04961..ca98eeedc2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -212,6 +212,13 @@ public File getObjectPath(File referencePath) { * @params * reference Pathname (maybe with magic suffix) to reference repo */ + public Boolean isParameterizedReferenceRepository(File reference) { + if (reference == null) { + return false; + } + return isParameterizedReferenceRepository(reference.getPath()); + } + public Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; @@ -243,6 +250,13 @@ public Boolean isParameterizedReferenceRepository(String reference) { * url URL of the repository being cloned, to help choose a * suitable parameterized reference repo subdirectory. */ + public File findParameterizedReferenceRepository(File reference, String url) { + if (reference == null) { + return reference; + } + return findParameterizedReferenceRepository(reference.getPath(), url); + } + public File findParameterizedReferenceRepository(String reference, String url) { if (reference == null || reference.isEmpty()) { return null; From 509c5ecd44fdc21f7119eda70d81197b53c20b7e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:48:13 +0100 Subject: [PATCH 019/155] [TMP] LegacyCompatibleGitAPIImpl.java: trace findParameterizedReferenceRepository() processing to stderr --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ca98eeedc2..f9f9bba49d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -287,6 +287,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } + System.err.println("reference='" + reference + "'\n" + + "url='" + url + "'\n" + + "urlNormalized='" + urlNormalized + "'\n"); + if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -310,6 +314,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { referencePath = null; // GC referencePath = new File(reference); } + System.err.println("reference after='" + reference + "'\n"); } if (!referencePath.exists()) { From 0b0fcbfceeb0fdd9de36d9e2abde0f8bd48201e0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:48:51 +0100 Subject: [PATCH 020/155] [TMP] WorkspaceWithRepo.java (test): trace localMirror() processing to stderr --- .../jenkinsci/plugins/gitclient/GitClientCloneTest.java | 7 +++++++ .../jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b0a986ebfb..873dda9bb3 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -286,12 +286,19 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); + + System.err.println("clone output:\n======\n" + messages + "\n======\n"); + assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index effea085aa..22bdb883a7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -82,10 +82,13 @@ String localMirror() throws IOException, InterruptedException { String localMirror(String cloneDirName) throws IOException, InterruptedException { File base = new File(".").getAbsoluteFile(); + System.err.println("=== Beginning to search for cloneDirName='" + cloneDirName + "' from " + base.getPath()); for (File f = base; f != null; f = f.getParentFile()) { + System.err.println("Looking for 'target' in " + f.getPath()); File targetDir = new File(f, "target"); if (targetDir.exists()) { File clone = new File(targetDir, cloneDirName); + System.err.println("Looking for cloneDirName " + cloneDirName + " in " + targetDir.getPath()); if (!clone.exists()) { /* Clone to a temporary directory then move the * temporary directory to the final destination @@ -96,8 +99,10 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * the final destination directory. */ Path tempClonePath = Files.createTempDirectory(targetDir.toPath(), "clone-"); + System.err.println("tempClonePath=" + tempClonePath + " => (FQPN)" + tempClonePath.toFile().getAbsolutePath()); cliGitCommand.run("clone", "--reference", f.getCanonicalPath(), "--mirror", repoURL, tempClonePath.toFile().getAbsolutePath()); if (!clone.exists()) { // Still a race condition, but a narrow race handled by Files.move() + System.err.println("moving tempClonePath to cloneDirName=" + cloneDirName); renameAndDeleteDir(tempClonePath, cloneDirName); } else { /* @@ -122,8 +127,11 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * deleteRecursive() will discard a clone that * 'lost the race'. */ + System.err.println("removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } + } else { + System.err.println("FOUND cloneDirName " + cloneDirName + " in " + targetDir.getPath()); } // Strip away the "/./" in "...git-client-plugin/./target/..." return Paths.get(clone.getPath()).normalize().toString(); From 6a9c79d8dc63dfbfb63df0182e489966b50d9696 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:07:37 +0100 Subject: [PATCH 021/155] LegacyCompatibleGitAPIImpl.java: refactor to use referenceExpanded to keep original reference intact, and as indicator to recreate referencePath object once for many cases --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f9f9bba49d..d84e2402da 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -291,6 +291,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); + String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -304,13 +305,16 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); - referencePath = null; // GC - referencePath = new File(reference); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - reference = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + } + } + + if (referenceExpanded != null) { + reference = referenceExpanded; referencePath = null; // GC referencePath = new File(reference); } From 264197188a27f41651bfb9458f753423903e5d61 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:09:08 +0100 Subject: [PATCH 022/155] LegacyCompatibleGitAPIImpl.java: add GIT_URL_FALLBACK and GIT_URL_SHA256_FALLBACK suffixes for using unsuffixed directory if expanded path points nowhere useful --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d84e2402da..e67276ea9c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -232,6 +232,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + return true; + } + + if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { + return true; + } + return false; } @@ -305,11 +313,50 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? + + // TODO: employ git submodules - there we can reliably match + // remote URLs (easily multiple) to particular modules, and + // yet keep separate git index directories per module with + // smaller scopes - much quicker to check out from than one + // huge combined repo. It would also be much more native to + // tools and custom scriptware that can be involved. + + // TODO: Config option whether to populate absent reference + // repos (If the expanded path does not have git repo data + // right now, populate it in the location expanded below) + // or update existing ones before pulling commits, and how + // to achieve that. Note that this is done by caller with + // their implementation of git, or in case of a build farm it + // is more likely to be a shared path only readable to Jenkins + // and its agents, so write-operations would be done by helper + // scripts that log into the shared storage server to populate + // or update reference repositories. Note that users may also + // want to run their own scripts to "populate" reference repos + // as symlinks to existing other repos, to support combined + // repo setup for different URLs pointing to same upstream, + // or storing multiple closely related forks together. + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { + // The safest option - fall back to parent directory if + // the expanded one does not have git repo data right now: + // it allows to gradually convert a big combined reference + // repository into smaller chunks without breaking builds. + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); + } + } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } } From c10231ca971f9856b43200d3c0cb2baab0364124 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:50:32 +0100 Subject: [PATCH 023/155] GitClientCloneTest.java: add cases for parameterized reference repo with fallback modes of operation --- .../plugins/gitclient/GitClientCloneTest.java | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 873dda9bb3..26f7953690 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -250,13 +250,46 @@ public void test_clone_reference_parameterized() throws Exception, IOException, handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: "), is(true)); - // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + // TODO: Actually construct the local filesystem path to match + // the literally substituted URL. Note this may not work for + // filesystems and operating systems that constrain character + // ranges usable for filenames (Windows at least) so failure + // to create the paths is an option to handle in test. + // Be sure to clean away this path at end of test, so that the + // test_clone_reference_parameterized_fallback() below is not + // confused - it expects this location to not exist. + // Skip: Missing if clone failed - currently would, with bogus + // path above and not yet pre-created path structure. //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); //assertAlternateFilePointsToLocalMirror(); //assertBranchesExist(testGitClient.getBranches(), "master"); //assertNoObjectsInRepository(); } + @Test + public void test_clone_reference_parameterized_fallback() throws Exception, IOException, InterruptedException { + // TODO: Currently we do not make paths which would invalidate + // this test, but note the test above might do just that later. + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_FALLBACK}").execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), + is(true)); + // With fallback mode, and nonexistent parameterized reference + // repository, and a usable repository in the common path (what + // remains if the parameterizing suffix is just discarded), + // this common path should be used. So it should overall behave + // same as the non-parameterized test_clone_reference() above. + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } + @Test public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); @@ -313,6 +346,64 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure } + @Test + public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ + String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); + + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}").execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + + System.err.println("clone output:\n======\n" + messages + "\n======\n"); + + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: ") && + (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + is(true)); + + // Barring filesystem errors, if we have the "custom" refrepo + // we expect it to be used (fallback mode is not triggered) + if (wsRefrepo != null) { + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From 201671907eb9bf49964dd022325da15fffe7ba18 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 22:14:56 +0100 Subject: [PATCH 024/155] LegacyCompatibleGitAPIImpl.java: refactor normalizeGitUrl() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e67276ea9c..b4eea9cdc7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -243,6 +243,32 @@ public Boolean isParameterizedReferenceRepository(String reference) { return false; } + /** There are many ways to spell an URL to the same repository even if + * using the same access protocol. This routine converts the "url" string + * in a way that helps us confirm whether two spellings mean same thing. + */ + public String normalizeGitUrl(String url, Boolean checkLocalPaths) { + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + if (!url.contains("://") && + !url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + if (checkLocalPaths) { + File urlPath = new File(url); + if (!urlPath.exists()) { + // Also not a subdirectory of current dir without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Assume it is not a path + urlNormalized = "ssh://" + urlNormalized; + } + } + + return urlNormalized; + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small @@ -282,18 +308,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // The actual reference repository directory may choose to have // original URL strings added as remotes (in case some are case // sensitive and different). - String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); - if (!url.contains("://") && - !url.startsWith("/") && - !url.startsWith(".") - ) { - // Not an URL with schema, not an absolute or relative pathname - File urlPath = new File(url); - if (!urlPath.exists()) { - // Also not a subdirectory without "./" prefix... - urlNormalized = "ssh://" + urlNormalized; - } - } + String urlNormalized = normalizeGitUrl(url, true); System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + From 71c0850d440690e672b8d1cb5af65745e97c2a1c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 23:07:01 +0100 Subject: [PATCH 025/155] [WIP] LegacyCompatibleGitAPIImpl.java: skeleton for GIT_SUBMODULES and GIT_SUBMODULES_FALLBACK support --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b4eea9cdc7..d1441d9e00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -240,6 +240,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_SUBMODULES}")) { + return true; + } + + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + return true; + } + return false; } @@ -373,6 +381,14 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + // Assuming the provided "reference" directory already hosts + // submodules, we use git tools to find the one subdir which + // has a registered remote URL equivalent (per normalization) + // to the provided "url". If there is no hit, the non-fallback + // mode suggests a new directory name to host the submodule + // (via same rules as for SHA256), and the fallback mode would + // return the main directory. } if (referenceExpanded != null) { From ab04ae7ae43da4aa8f1484c962d2264ce1dd1768 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 01:22:35 +0100 Subject: [PATCH 026/155] Introduce getRemoteUrls() and getRemotePushUrls() --- .../plugins/gitclient/CliGitAPIImpl.java | 42 ++++++++++++++++++ .../plugins/gitclient/GitClient.java | 22 ++++++++++ .../plugins/gitclient/JGitAPIImpl.java | 44 +++++++++++++++++++ .../plugins/gitclient/RemoteGitImpl.java | 10 +++++ 4 files changed, 118 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index b8c011a08c..c2b7873e0f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1584,6 +1584,48 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr return StringUtils.trim(firstLine(result)); } + /** {@inheritDoc} + * + * Note: unlike the version in JGitAPIImpl, this one does not filter + * ASCII or Private URLs, and just conveys what git config contains. + */ + @Override + public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { + String result = launchCommand( "config", "--local", "--list" ); + Map uriNames = new HashMap<>(); + for (String line : result.split("\\R+")) { + line = StringUtils.trim(line); + if (!line.startsWith("remote.") || !line.contains(".url=")) { + continue; + } + + String remoteName = StringUtils.substringBetween(line, "remote.", ".url="); + String remoteUri = StringUtils.substringAfter(line, ".url="); + + // If uri String values end up identical, Map only stores one entry + uriNames.put(remoteUri, remoteName); + } + return uriNames; + } + + /** {@inheritDoc} */ + @Override + public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { + String result = launchCommand( "config", "--local", "--list" ); + Map uriNames = new HashMap<>(); + for (String line : result.split("\\R+")) { + line = StringUtils.trim(line); + if (!line.startsWith("remote.") || !line.contains(".pushurl=")) { + continue; + } + + String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl="); + String remoteUri = StringUtils.substringAfter(line, ".pushurl="); + uriNames.put(remoteUri, remoteName); + } + return uriNames; + } + /** {@inheritDoc} */ @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 644a4ff236..498d73bdc1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -211,6 +211,28 @@ public interface GitClient { */ String getRemoteUrl(String name) throws GitException, InterruptedException; + /** + * getRemoteUrls. + * + * @return a Map where String keys represent URIs (with and + * without passwords, if any; ASCII or not, if + * applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + public Map getRemoteUrls() throws GitException, InterruptedException; + + /** + * getRemotePushUrls. + * + * @return a Map where String keys represent push-only URIs + * (with and without passwords, if any; ASCII or not, + * if applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + public Map getRemotePushUrls() throws GitException, InterruptedException; + /** * For a given repository, set a remote's URL * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a83981b144..cf609ebcb9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -911,6 +911,50 @@ public String getRemoteUrl(String name) throws GitException { } } + /** {@inheritDoc} */ + @Override + public Map getRemoteUrls() throws GitException, InterruptedException { + Map uriNames = new HashMap<>(); + try (Repository repo = getRepository()) { + Config c = repo.getConfig(); + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { + String remoteName = rc.getName(); + for (URIish u : rc.getURIs()) { + // If uri String values end up identical, Map only stores one entry + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException ue) { + throw new GitException(ue.toString()); + } + return uriNames; + } + + /** {@inheritDoc} */ + @Override + public Map getRemotePushUrls() throws GitException, InterruptedException { + Map uriNames = new HashMap<>(); + try (Repository repo = getRepository()) { + Config c = repo.getConfig(); + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { + String remoteName = rc.getName(); + for (URIish u : rc.getPushURIs()) { + // If uri String values end up identical, Map only stores one entry + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException ue) { + throw new GitException(ue.toString()); + } + return uriNames; + } + /** * getRepository. * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index 63b504168d..68a116010c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -294,6 +294,16 @@ public String getRemoteUrl(String name) throws GitException, InterruptedExceptio return proxy.getRemoteUrl(name); } + /** {@inheritDoc} */ + public Map getRemoteUrls() throws GitException, InterruptedException { + return proxy.getRemoteUrls(); + } + + /** {@inheritDoc} */ + public Map getRemotePushUrls() throws GitException, InterruptedException { + return proxy.getRemotePushUrls(); + } + /** {@inheritDoc} */ public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { proxy.setRemoteUrl(name, url); From 179f5c192919b1b1f15fc48c8ab49744617e39ea Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 02:08:24 +0100 Subject: [PATCH 027/155] LegacyCompatibleGitAPIImpl.java: normalizeGitUrl(): revise for local path names --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d1441d9e00..a7639ecae4 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -257,20 +259,30 @@ public Boolean isParameterizedReferenceRepository(String reference) { */ public String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); - if (!url.contains("://") && - !url.startsWith("/") && - !url.startsWith(".") - ) { - // Not an URL with schema, not an absolute or relative pathname - if (checkLocalPaths) { - File urlPath = new File(url); - if (!urlPath.exists()) { - // Also not a subdirectory of current dir without "./" prefix... + if (!url.contains("://")) { + if (!url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + if (checkLocalPaths) { + File urlPath = new File(url); + if (urlPath.exists()) { + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + } else { + // Also not a subdirectory of current dir without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Assume it is not a path urlNormalized = "ssh://" + urlNormalized; } } else { - // Assume it is not a path - urlNormalized = "ssh://" + urlNormalized; + // Looks like a local path + if (url.startsWith("/")) { + urlNormalized = "file://" + urlNormalized; + } else { + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + } } } From 7c0ad4f4de7edc8e7028692f1c0a1e1ba276837b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 02:17:06 +0100 Subject: [PATCH 028/155] [WIP] LegacyCompatibleGitAPIImpl.java: skeleton for GIT_SUBMODULES and GIT_SUBMODULES_FALLBACK support --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index a7639ecae4..cf1a0b1d1a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -401,6 +401,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // mode suggests a new directory name to host the submodule // (via same rules as for SHA256), and the fallback mode would // return the main directory. + // For each current workspace (recurse): + // try { getSubmodules("HEAD") ... } => List filtered for "commit" items + // getRemoteUrls() => Map } if (referenceExpanded != null) { From ead03a60380837b161733b2f2690880362781d91 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 09:50:11 +0100 Subject: [PATCH 029/155] CliGitAPIImpl.java: extend getRemote{,Push}Urls() with filters for ACSII and for "safe" URLs without username/password, to match jGit version --- .../plugins/gitclient/CliGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index c2b7873e0f..b1875b757b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1584,11 +1584,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr return StringUtils.trim(firstLine(result)); } - /** {@inheritDoc} - * - * Note: unlike the version in JGitAPIImpl, this one does not filter - * ASCII or Private URLs, and just conveys what git config contains. - */ + /** {@inheritDoc} */ @Override public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { String result = launchCommand( "config", "--local", "--list" ); @@ -1604,6 +1600,14 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr // If uri String values end up identical, Map only stores one entry uriNames.put(remoteUri, remoteName); + + try { + URI u = new URI(remoteUri); + uriNames.put(u.toASCIIString(), remoteName); + URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); + uriNames.put(uSafe.toString(), remoteName); + uriNames.put(uSafe.toASCIIString(), remoteName); + } catch (URISyntaxException ue) {} // ignore, move along } return uriNames; } @@ -1622,6 +1626,14 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl="); String remoteUri = StringUtils.substringAfter(line, ".pushurl="); uriNames.put(remoteUri, remoteName); + + try { + URI u = new URI(remoteUri); + uriNames.put(u.toASCIIString(), remoteName); + URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); + uriNames.put(uSafe.toString(), remoteName); + uriNames.put(uSafe.toASCIIString(), remoteName); + } catch (URISyntaxException ue) {} // ignore, move along } return uriNames; } From 5fde3c916b70c018c7094776d93fde581654944b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:01:14 +0100 Subject: [PATCH 030/155] GitClientCloneTest.java: if we expect particular wsRefrepo spelling, note it in assertion stacktrace --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 26f7953690..0bfcd72c75 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -332,7 +332,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages, + assertThat("Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , @@ -388,7 +389,8 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages, + assertThat("Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , From 6c7374ccb4f039c94eefe92ffb6e68048646c6a2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:04:50 +0100 Subject: [PATCH 031/155] GitClientCloneTest.java: for fallback case, expect wsRefrepoBase itself --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 0bfcd72c75..426719ce78 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -390,10 +390,9 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); assertThat("Reference repo name-parsing logged in: " + messages + - (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , + "\n...and replaced with: '" + wsRefrepoBase + "'", handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: ") && - (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase + "'"), is(true)); // Barring filesystem errors, if we have the "custom" refrepo From a2cf84f23d9cdcf9b2c65223ef2fd6534b6e08f8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:11:10 +0100 Subject: [PATCH 032/155] GitClientCloneTest.java: use the LegacyCompatibleGitAPIImpl.normalizeGitUrl() directly --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- .../plugins/gitclient/GitClientCloneTest.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index cf1a0b1d1a..8c998c3b8b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -257,7 +257,7 @@ public Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ - public String normalizeGitUrl(String url, Boolean checkLocalPaths) { + public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { if (!url.startsWith("/") && diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 426719ce78..fcfd9149ff 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -293,8 +293,10 @@ public void test_clone_reference_parameterized_fallback() throws Exception, IOEx @Test public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); - /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ - String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ @@ -350,8 +352,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce @Test public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); - /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ - String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ From 823bd0f0697d4bffc6a1a82ef9c5a59833c75ac3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:49:25 +0100 Subject: [PATCH 033/155] GitClientCloneTest.java: for fallback case, expect wsRefrepoBase itself (might have or not the SHA dir after it) --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index fcfd9149ff..5bcb91bf80 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -393,10 +393,12 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); + // Note: we do not expect the closing single quote after wsRefrepoBase + // because other tests might pollute our test area, and SHA dir is there assertThat("Reference repo name-parsing logged in: " + messages + - "\n...and replaced with: '" + wsRefrepoBase + "'", + "\n...and replaced with: '" + wsRefrepoBase, handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase + "'"), + handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), is(true)); // Barring filesystem errors, if we have the "custom" refrepo From 0e5e0cd9673ded2fee8cf3b4a49544d156cf4d86 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 12 Dec 2020 01:31:31 +0100 Subject: [PATCH 034/155] LegacyCompatibleGitAPIImpl.java: skeleton for getSubmodulesUrls() as needed for GIT_SUBMODULES --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 54 ++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8c998c3b8b..4186c00a4f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -24,8 +24,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -289,6 +291,45 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { return urlNormalized; } + /** Find referenced URLs in this repo and its submodules, recursively. + * Current primary use is for parameterized refrepo/${GIT_SUBMODULES} + * + * @return a Set of (unique) String arrays, representing: + * [0] directory of nested submodule (relative to current workspace root) + * The current workspace would be listed as directory "" and consumers + * should check these entries last if they care for most-specific hits + * with smaller-scope reference repositories. + * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several entries per remote + * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames fully qualified) + * [3] remoteName as defined in that nested submodule + * + * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) + * which we want to find if it is not null - so stop and + * return just hits for it as soon as we have something. + */ + public Set getSubmodulesUrls(String needle) { + Set result = new HashSet<>(); + // For each current workspace (recurse?): + // subGit(subdir) + // try { getSubmodules("HEAD") ... } => List filtered for "commit" items + // getRemoteUrls() => Map + + // If needle is not null, look in SHA256 named dir that can match it; + // note that this use-case might pan out also if "this" repo is bare + // and can not have "proper" git submodules. If it is bare, look at + // remote URLs in current dir after the guessed subdir and return then. + + // Otherwise first dig into submodules, when there is no deeper to drill, + // report remote URLs and step back from recursion. + return result; + } + + /** See above. With null, returns all we can find (slower) for the caller + * to parse */ + public Set getSubmodulesUrls() { + return getSubmodulesUrls(null); + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small @@ -397,13 +438,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) - // to the provided "url". If there is no hit, the non-fallback - // mode suggests a new directory name to host the submodule - // (via same rules as for SHA256), and the fallback mode would - // return the main directory. - // For each current workspace (recurse): - // try { getSubmodules("HEAD") ... } => List filtered for "commit" items - // getRemoteUrls() => Map + // to the provided "url". + // If there is no hit, the non-fallback mode suggests a new + // directory name to host the submodule (same rules as SHA), + // and the fallback mode would return the main directory. + + // getSubmodulesUrls(urlNormalized) => dirname if any } if (referenceExpanded != null) { From 172e40eeb2bff56b1415d29b6738601ce07d0d84 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 14 Dec 2020 10:19:40 +0100 Subject: [PATCH 035/155] LegacyCompatibleGitAPIImpl.java: update comments for ideas about submodule searches --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 4186c00a4f..50d638887a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -299,8 +299,10 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * The current workspace would be listed as directory "" and consumers * should check these entries last if they care for most-specific hits * with smaller-scope reference repositories. - * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several entries per remote - * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames fully qualified) + * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several + * entries per remote + * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames + * fully qualified) * [3] remoteName as defined in that nested submodule * * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) @@ -308,19 +310,34 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * return just hits for it as soon as we have something. */ public Set getSubmodulesUrls(String needle) { + // Keep track of where we've already looked in the "result" Set, to + // avoid looking in same places (different strategies below) twice. + // And eventually return this Set or part of it as the answer. Set result = new HashSet<>(); - // For each current workspace (recurse?): - // subGit(subdir) + + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? // try { getSubmodules("HEAD") ... } => List filtered for "commit" items // getRemoteUrls() => Map - // If needle is not null, look in SHA256 named dir that can match it; - // note that this use-case might pan out also if "this" repo is bare - // and can not have "proper" git submodules. If it is bare, look at - // remote URLs in current dir after the guessed subdir and return then. - - // Otherwise first dig into submodules, when there is no deeper to drill, - // report remote URLs and step back from recursion. + // If needle is not null, first look perhaps in the subdir(s) named + // with base-name of the URL with and without a ".git" suffix, then + // in SHA256 named dir that can match it; note that this use-case + // might pan out also if "this" repo is bare and can not have "proper" + // git submodules - but was prepared for our other options. + + // If current repo *is* bare (can't have proper submodules) and the + // needle is not null, follow up with: + // * Maybe also direct child dirs that have a ".git" FS object inside?.. + // * Look at remote URLs in current dir after the guessed subdir and + // return then. + + // If current dir does have submodules, first dig into submodules, + // when there is no deeper to drill, report remote URLs and step + // back from recursion. This way we have least specific repo last, + // if several have the replica (assuming the first hits are smaller + // scopes). return result; } From d6cebe891fdf4a204b90ffdb6aa3b7ea8ed3db77 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 14 Dec 2020 14:03:44 +0100 Subject: [PATCH 036/155] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls(): implement the search for "needle" in bare repo case --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 128 +++++++++++++++++- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 50d638887a..0f6b0a3ce6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -25,6 +25,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -313,7 +314,15 @@ public Set getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. - Set result = new HashSet<>(); + Set result = new LinkedHashSet<>(); // Retain order of insertion + File f = null; + + Boolean isBare = false; + try { + isBare = this.isBareRepository(); + } catch (InterruptedException e) { + isBare = false; // At least try to look into submodules... + } // For each current workspace (recurse or big loop in same context?): // public GitClient subGit(String subdir) => would this.subGit(...) @@ -326,12 +335,119 @@ public Set getSubmodulesUrls(String needle) { // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. + if (needle != null && !needle.isEmpty()) { + int sep = needle.lastIndexOf("/"); + String needleNorm = normalizeGitUrl(needle, true); + String needleBasename; + if (sep < 0) { + needleBasename = needle; + } else { + needleBasename = needle.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + // Try with the basename without .git extension, and then with one. + ArrayList arrDirnames = new ArrayList(); + arrDirnames.add(needleBasename); + arrDirnames.add(needleBasename + ".git"); + String needleBasenameLC = needleBasename.toLowerCase(); + if (!needleBasenameLC.equals(needleBasename)) { + // Retry with lowercased dirname + arrDirnames.add(needleBasenameLC); + arrDirnames.add(needleBasenameLC + ".git"); + } + String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + arrDirnames.add(needleSha); + arrDirnames.add(needleSha + ".git"); + + for (String dirname : arrDirnames) { + f = new File(".", dirname); + if (f.exists() && f.isDirectory()) { + try { + //LegacyCompatibleGitAPIImpl? + GitClient g = this.subGit(needleBasename); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new HashSet<>(); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } + } - // If current repo *is* bare (can't have proper submodules) and the - // needle is not null, follow up with: - // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdir and - // return then. + // If current repo *is* bare (can't have proper submodules) and the + // needle is not null, follow up with: + // * Maybe also direct child dirs that have a ".git" FS object inside?.. + // * Look at remote URLs in current dir after the guessed subdir and + // return then. + if (isBare) { + arrDirnames.clear(); + + // Check subdirs that are git workspaces + Set checkedDirs = new HashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + File[] directories = new File(".").listFiles(File::isDirectory); + for (File dir : directories) { + f = new File(dir, ".git"); + if (f.exists()) { // May be a file or directory... or symlink to those... + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } + } + } + + // Finally check parent dir + arrDirnames.add("."); + + for (String dirname : arrDirnames) { + f = new File(".", dirname); + if (f.exists() && f.isDirectory()) { + try { + //LegacyCompatibleGitAPIImpl? + GitClient g = this.subGit(needleBasename); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new HashSet<>(); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + // Commented away: currently we exit after this loop. + //result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } + } + + // Nothing found, and for bare top-level repo not much to search for + return new HashSet<>(); + } + } else { + // ...and if there is no needle? + /* if (isBare) { ... } */ + // fall through currently, let git decide if it has submodules here and now + } // If current dir does have submodules, first dig into submodules, // when there is no deeper to drill, report remote URLs and step From 864e34febd4f936d4dfeca1a51084cdf49179759 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:04:13 +0100 Subject: [PATCH 037/155] LegacyCompatibleGitAPIImpl.java: move arrDirNames declaration higher in scope --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0f6b0a3ce6..cb219ea427 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -316,6 +316,8 @@ public Set getSubmodulesUrls(String needle) { // And eventually return this Set or part of it as the answer. Set result = new LinkedHashSet<>(); // Retain order of insertion File f = null; + // Helper list storage in loops below + ArrayList arrDirnames = new ArrayList(); Boolean isBare = false; try { @@ -347,7 +349,6 @@ public Set getSubmodulesUrls(String needle) { needleBasename = needleBasename.replaceAll(".git$", ""); // Try with the basename without .git extension, and then with one. - ArrayList arrDirnames = new ArrayList(); arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); String needleBasenameLC = needleBasename.toLowerCase(); From 55fda14f7673b4fb82b169b180577d12a1ab8b72 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:04:56 +0100 Subject: [PATCH 038/155] LegacyCompatibleGitAPIImpl.java: update commented thoughts --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index cb219ea427..de42fbeb00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -390,12 +390,14 @@ public Set getSubmodulesUrls(String needle) { // If current repo *is* bare (can't have proper submodules) and the // needle is not null, follow up with: // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdir and - // return then. + // * Look at remote URLs in current dir after the guessed subdirs failed, + // and return then. if (isBare) { arrDirnames.clear(); // Check subdirs that are git workspaces + // TODO: Refactor to avoid lookups of dirs that may prove not + // needed in the end (aim for less I/Os to find the goal) Set checkedDirs = new HashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); @@ -442,6 +444,7 @@ public Set getSubmodulesUrls(String needle) { } // Nothing found, and for bare top-level repo not much to search for + // Or maybe also fall through to let git decide? return new HashSet<>(); } } else { From 3e81723af9dc9def9b904cbf481c732db8ce822c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:05:44 +0100 Subject: [PATCH 039/155] LegacyCompatibleGitAPIImpl.java: prepare to parse submodule workspaces --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index de42fbeb00..b2556bf144 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -326,12 +326,6 @@ public Set getSubmodulesUrls(String needle) { isBare = false; // At least try to look into submodules... } - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // try { getSubmodules("HEAD") ... } => List filtered for "commit" items - // getRemoteUrls() => Map - // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case @@ -453,6 +447,22 @@ public Set getSubmodulesUrls(String needle) { // fall through currently, let git decide if it has submodules here and now } + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // getRemoteUrls() => Map + arrDirnames.clear(); + + // Check subdirs that are git workspaces + Set checkedDirs = new HashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + // If current dir does have submodules, first dig into submodules, // when there is no deeper to drill, report remote URLs and step // back from recursion. This way we have least specific repo last, From ff65341e1c15809c906c61a81dae05cb5e21e348 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:06:29 +0100 Subject: [PATCH 040/155] LegacyCompatibleGitAPIImpl.java: uncomment needle tracking in a loop that we might not-abort after all --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b2556bf144..77562022a0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -428,8 +428,7 @@ public Set getSubmodulesUrls(String needle) { // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - // Commented away: currently we exit after this loop. - //result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From 122746fe54e74a3c6f2565cff189ed6657a86d70 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 02:28:37 +0100 Subject: [PATCH 041/155] LegacyCompatibleGitAPIImpl.java: fix @param usage --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 77562022a0..2af842e0fd 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -214,8 +214,7 @@ public File getObjectPath(File referencePath) { /** Handle magic strings in the reference pathname to sort out patterns * classified as evaluated by parametrization, as handled below * - * @params - * reference Pathname (maybe with magic suffix) to reference repo + * @param reference Pathname (maybe with magic suffix) to reference repo */ public Boolean isParameterizedReferenceRepository(File reference) { if (reference == null) { @@ -486,10 +485,9 @@ public Set getSubmodulesUrls() { * * At some point this plugin might also maintain that filesystem structure. * - * @params - * reference Pathname (maybe with magic suffix) to reference repo - * url URL of the repository being cloned, to help choose a - * suitable parameterized reference repo subdirectory. + * @param reference Pathname (maybe with magic suffix) to reference repo + * @param url URL of the repository being cloned, to help choose a + * suitable parameterized reference repo subdirectory. */ public File findParameterizedReferenceRepository(File reference, String url) { if (reference == null) { From bf978eec0544589e7781085f6664a0f0a29dfd0f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 02:48:36 +0100 Subject: [PATCH 042/155] LegacyCompatibleGitAPIImpl.java: implement handling output of getSubmodulesUrls() in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2af842e0fd..716462daab 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -406,7 +406,7 @@ public Set getSubmodulesUrls(String needle) { } } - // Finally check parent dir + // Finally check pattern's parent dir arrDirnames.add("."); for (String dirname : arrDirnames) { @@ -583,11 +583,32 @@ public File findParameterizedReferenceRepository(String reference, String url) { // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) // to the provided "url". - // If there is no hit, the non-fallback mode suggests a new - // directory name to host the submodule (same rules as SHA), - // and the fallback mode would return the main directory. - // getSubmodulesUrls(urlNormalized) => dirname if any + Set subEntries = getSubmodulesUrls(urlNormalized); + if (!subEntries.isEmpty()) { + // Normally we should only have one entry here, as sorted by the routine + referenceExpanded = subEntries.iterator().next()[0]; + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + } + } else { + // If there is no hit, the non-fallback mode suggests a new + // directory name to host the submodule (same rules as SHA), + // and the fallback mode would return the main directory. + if (reference.endsWith("/${GIT_SUBMODULES}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + } + } + else if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + } + } } if (referenceExpanded != null) { From aacc0cdd13f8a767a3561063d576a860d9601b83 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 03:14:36 +0100 Subject: [PATCH 043/155] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls() / findParameterizedReferenceRepository(): deal with LinkedHashSet instead of plain Set to maintain insertion order as the priority --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 716462daab..ef2796ba68 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -309,11 +309,11 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * which we want to find if it is not null - so stop and * return just hits for it as soon as we have something. */ - public Set getSubmodulesUrls(String needle) { + public LinkedHashSet getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. - Set result = new LinkedHashSet<>(); // Retain order of insertion + LinkedHashSet result = new LinkedHashSet<>(); // Retain order of insertion File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); @@ -365,7 +365,7 @@ public Set getSubmodulesUrls(String needle) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new HashSet<>(); + result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); return result; } @@ -391,7 +391,7 @@ public Set getSubmodulesUrls(String needle) { // Check subdirs that are git workspaces // TODO: Refactor to avoid lookups of dirs that may prove not // needed in the end (aim for less I/Os to find the goal) - Set checkedDirs = new HashSet<>(); + LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } @@ -420,7 +420,7 @@ public Set getSubmodulesUrls(String needle) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new HashSet<>(); + result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); return result; } @@ -437,7 +437,7 @@ public Set getSubmodulesUrls(String needle) { // Nothing found, and for bare top-level repo not much to search for // Or maybe also fall through to let git decide? - return new HashSet<>(); + return new LinkedHashSet<>(); } } else { // ...and if there is no needle? @@ -456,7 +456,7 @@ public Set getSubmodulesUrls(String needle) { arrDirnames.clear(); // Check subdirs that are git workspaces - Set checkedDirs = new HashSet<>(); + LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } @@ -471,7 +471,7 @@ public Set getSubmodulesUrls(String needle) { /** See above. With null, returns all we can find (slower) for the caller * to parse */ - public Set getSubmodulesUrls() { + public LinkedHashSet getSubmodulesUrls() { return getSubmodulesUrls(null); } @@ -584,7 +584,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // has a registered remote URL equivalent (per normalization) // to the provided "url". - Set subEntries = getSubmodulesUrls(urlNormalized); + LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted by the routine referenceExpanded = subEntries.iterator().next()[0]; From 1a8306aec5326e4723a09ad196d6291e7c241f34 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 03:16:38 +0100 Subject: [PATCH 044/155] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): comment a TODO in case there are several hits in the getSubmodulesUrls() return --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ef2796ba68..f92f058394 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -587,6 +587,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted by the routine + // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; if (getObjectPath(referenceExpanded) == null) { // chop it off, use main directory From 769a2d4b6c28cb00934e4148a78557e3f2a9785d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:22:26 +0100 Subject: [PATCH 045/155] LegacyCompatibleGitAPIImpl.java: log-tracing for getSubmodulesUrls() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f92f058394..fc9c04df99 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -359,11 +359,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? + System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); + System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -414,11 +416,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? + System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); + System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -589,11 +593,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Normally we should only have one entry here, as sorted by the routine // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; + System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); if (getObjectPath(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } } else { + System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as SHA), // and the fallback mode would return the main directory. From 654362a0218fb1114be68c3cc163f4139c9323cf Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:22:51 +0100 Subject: [PATCH 046/155] LegacyCompatibleGitAPIImpl.java: for now look in subdirs regardless of isbare --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index fc9c04df99..89148cc84e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -387,7 +387,8 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // * Maybe also direct child dirs that have a ".git" FS object inside?.. // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (isBare) { + // TODO: Remove "|| true" when other logic fo look for real submodules is complete + if (isBare || true) { arrDirnames.clear(); // Check subdirs that are git workspaces @@ -441,7 +442,9 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Nothing found, and for bare top-level repo not much to search for // Or maybe also fall through to let git decide? - return new LinkedHashSet<>(); + if (isBare) { + return new LinkedHashSet<>(); + } } } else { // ...and if there is no needle? @@ -459,7 +462,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // getRemoteUrls() => Map arrDirnames.clear(); - // Check subdirs that are git workspaces + // TODO: Check subdirs that are git workspaces, and remove "|| true" above LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); From 5b6ff6d699ef8e042a8283e47409be76226325e7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:23:01 +0100 Subject: [PATCH 047/155] LegacyCompatibleGitAPIImpl.java: comment updated --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 89148cc84e..82c8f1f824 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -447,7 +447,8 @@ public LinkedHashSet getSubmodulesUrls(String needle) { } } } else { - // ...and if there is no needle? + // ...and if there is no needle? Return info on all git remotes + // found under this directory... /* if (isBare) { ... } */ // fall through currently, let git decide if it has submodules here and now } From 8d8514c93ab67738bc09e2baa09b8fa92e14a63c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:01:25 +0100 Subject: [PATCH 048/155] LegacyCompatibleGitAPIImpl.java: rename getObjectPath() to less misleading getObjectsFile(), and varnames inside it too --- .../plugins/gitclient/CliGitAPIImpl.java | 2 +- .../plugins/gitclient/JGitAPIImpl.java | 2 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 40 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index b1875b757b..d6dda9cdaa 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -784,7 +784,7 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File objectsPath = getObjectPath(referencePath); + File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index cf609ebcb9..a46330a4ca 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1473,7 +1473,7 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File objectsPath = getObjectPath(referencePath); + File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 82c8f1f824..69fb29ffe2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -171,44 +171,44 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, * paths, while a not-null return value is instantly usable by * some code which plays with git under its hood. */ - public File getObjectPath(String reference) { + public File getObjectsFile(String reference) { if (reference == null || reference.isEmpty()) { return null; } - return getObjectPath(new File(reference)); + return getObjectsFile(new File(reference)); } - public File getObjectPath(File referencePath) { - if (referencePath == null) { - return referencePath; + public File getObjectsFile(File reference) { + if (reference == null) { + return reference; } - if (!referencePath.exists()) + if (!reference.exists()) return null; - if (!referencePath.isDirectory()) + if (!reference.isDirectory()) return null; - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (objectsPath == null) { - return objectsPath; // Some Java error, could not make object from the paths involved + // reference pathname can either point to a normal or a base repository + File objects = new File(reference, ".git/objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved } - if (!objectsPath.isDirectory()) { + if (!objects.isDirectory()) { // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - if (objectsPath == null) { - return objectsPath; // Some Java error, could not make object from the paths involved + objects = new File(reference, "objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved } } - if (!objectsPath.isDirectory()) + if (!objects.isDirectory()) return null; // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath - return objectsPath; + return objects; } /** Handle magic strings in the reference pathname to sort out patterns @@ -576,7 +576,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } @@ -598,7 +598,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } @@ -610,7 +610,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } From 925eabae288c8607865436bb7ba9627a42dfb74c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:04:05 +0100 Subject: [PATCH 049/155] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() fallbacks, quick-check for ".git" dirname existence and usability too --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 69fb29ffe2..2a8a8d6034 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -576,13 +576,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } @@ -598,7 +598,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } @@ -610,7 +610,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } From a103bdda97f7e0a394d5c8d05b3bbb1410d7b9a2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:05:08 +0100 Subject: [PATCH 050/155] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() move GIT_URL_FALLBACK closer to GIT_URL --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2a8a8d6034..5056f51dc6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -232,11 +232,11 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } - if (reference.endsWith("/${GIT_URL_SHA256}")) { + if (reference.endsWith("/${GIT_URL_FALLBACK}")) { return true; } - if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + if (reference.endsWith("/${GIT_URL_SHA256}")) { return true; } @@ -565,6 +565,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { // or storing multiple closely related forks together. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); + } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); + } } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", @@ -580,12 +586,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } - } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); - } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From aaaef9c0130a33907c15febbcdcb19e407046d21 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:54:15 +0100 Subject: [PATCH 051/155] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() add a GIT_URL_BASENAME/GIT_URL_BASENAME_FALLBACK option --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5056f51dc6..063e67a218 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -244,6 +244,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + return true; + } + + if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + return true; + } + if (reference.endsWith("/${GIT_SUBMODULES}")) { return true; } @@ -586,6 +594,28 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } + } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + // This may be the more portable solution with regard to filesystems + int sep = urlNormalized.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = urlNormalized; + } else { + needleBasename = urlNormalized.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + } + } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From a77dab9df9abf6efe292e08a0bf4e801de41cd16 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:54:43 +0100 Subject: [PATCH 052/155] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() revise the GIT_SUBMODULES/GIT_SUBMODULES_FALLBACK handling --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 063e67a218..e8f19a237b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -624,13 +624,24 @@ public File findParameterizedReferenceRepository(String reference, String url) { LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { - // Normally we should only have one entry here, as sorted by the routine - // TODO: If several entries are present after all, iterate until first existing hit - referenceExpanded = subEntries.iterator().next()[0]; + // Normally we should only have one entry here, as sorted + // by the routine, and prefer that first option if a new + // reference repo would have to be made (and none exists). + // If several entries are present after all, iterate until + // first existing hit and return the first entry otherwise. + for (String[] subEntry : subEntries) { + if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { + referenceExpanded = subEntry[0]; + break; + } + } + if (referenceExpanded == null) { + referenceExpanded = subEntries.iterator().next()[0]; + } System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } } else { System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); @@ -640,12 +651,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); - } } - else if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } From d664eb348d209a26f6e5bc8afe061c6598aece9e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:29:05 +0100 Subject: [PATCH 053/155] LegacyCompatibleGitAPIImpl.java: pass not-normalized URL from findParameterizedReferenceRepository() to getSubmodulesUrls(), in case users manually cloned refrepos with whatever casing they use throughout --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e8f19a237b..7256ab735e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -313,9 +313,10 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * - * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) - * which we want to find if it is not null - so stop and - * return just hits for it as soon as we have something. + * @param needle - an URL which (or its normalized variant coming from + * normalizeGitUrl(url, true)) we want to find: + * if it is not null - then stop and return just hits + * for it as soon as we have something. */ public LinkedHashSet getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to @@ -340,16 +341,26 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // git submodules - but was prepared for our other options. if (needle != null && !needle.isEmpty()) { int sep = needle.lastIndexOf("/"); - String needleNorm = normalizeGitUrl(needle, true); String needleBasename; if (sep < 0) { needleBasename = needle; } else { needleBasename = needle.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".git$", ""); + needleBasename = needleBasename.replaceAll(".[Gg][Ii][Tt]$", ""); + + String needleNorm = normalizeGitUrl(needle, true); + sep = needleNorm.lastIndexOf("/"); + String needleNormBasename; + if (sep < 0) { + needleNormBasename = needleNorm; + } else { + needleNormBasename = needleNorm.substring(sep + 1); + } + needleNormBasename = needleNormBasename.replaceAll(".git$", ""); // Try with the basename without .git extension, and then with one. + // First we try the caller-provided string casing, then normalized. arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); String needleBasenameLC = needleBasename.toLowerCase(); @@ -358,6 +369,11 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleBasenameLC); arrDirnames.add(needleBasenameLC + ".git"); } + if (!needleNormBasename.equals(needleBasenameLC)) { + arrDirnames.add(needleNormBasename); + arrDirnames.add(needleNormBasename + ".git"); + } + String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); @@ -622,7 +638,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // has a registered remote URL equivalent (per normalization) // to the provided "url". - LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); + // Note: we pass "url" here, the routine differentiates original + // naming vs. normalization. + LinkedHashSet subEntries = getSubmodulesUrls(url); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From e29d9478c7cfce95329b0e58f24507f2c8262259 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:29:45 +0100 Subject: [PATCH 054/155] LegacyCompatibleGitAPIImpl.java: check also not-normalized URL in findParameterizedReferenceRepository() GIT_URL_BASENAME, in case users manually cloned refrepos with whatever casing they use throughout --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7256ab735e..3af1194aa2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -612,12 +612,14 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems - int sep = urlNormalized.lastIndexOf("/"); + // First try with original user-provided casing of the URL (if local + // dirs were cloned manually) + int sep = url.lastIndexOf("/"); String needleBasename; if (sep < 0) { - needleBasename = urlNormalized; + needleBasename = url; } else { - needleBasename = urlNormalized.substring(sep + 1); + needleBasename = url.substring(sep + 1); } needleBasename = needleBasename.replaceAll(".git$", ""); @@ -627,11 +629,34 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory + if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); } } + + if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // Retry with automation-ready normalized URL + sep = urlNormalized.lastIndexOf("/"); + if (sep < 0) { + needleBasename = urlNormalized; + } else { + needleBasename = urlNormalized.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + } + } + } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From 18166325deddaa3e7ec7f2efb328acecaccf583f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:50:36 +0100 Subject: [PATCH 055/155] LegacyCompatibleGitAPIImpl.java: only add ".git" to reference if it is not there yet (might be from *_FALLBACK cases) --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 3af1194aa2..35ee5925d9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -710,7 +710,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { System.err.println("reference after='" + reference + "'\n"); } - if (!referencePath.exists()) { + if (!referencePath.exists() && !reference.endsWith(".git")) { // Normalize the URLs with or without .git suffix to // be served by same dir with the refrepo contents reference += ".git"; From 8b693fd3e2be9ff141dd2b1e97565f6b84cac669 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:51:12 +0100 Subject: [PATCH 056/155] LegacyCompatibleGitAPIImpl.java: fix tracing into private LOGGER instead of stderr (except cases currently anticipated by unit-tests) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 35ee5925d9..67262b5cbb 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -43,6 +45,8 @@ */ abstract class LegacyCompatibleGitAPIImpl extends AbstractGitAPIImpl implements IGitAPI { + private static final Logger LOGGER = Logger.getLogger(LegacyCompatibleGitAPIImpl.class.getName()); + /** * isBareRepository. * @@ -296,6 +300,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { } } + LOGGER.log(Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -383,13 +388,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -441,13 +446,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -547,6 +552,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); @@ -681,13 +687,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (referenceExpanded == null) { referenceExpanded = subEntries.iterator().next()[0]; } - System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } } else { - System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as SHA), // and the fallback mode would return the main directory. @@ -707,6 +713,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { referencePath = null; // GC referencePath = new File(reference); } + + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); } From e49b985cdb388a27f9932e9e060d06e99c28f227 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:07:00 +0100 Subject: [PATCH 057/155] LegacyCompatibleGitAPIImpl.java: trace looking for the needle in arrDirnames --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 67262b5cbb..58b5b59ffc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -383,13 +383,16 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + for (String dirname : arrDirnames) { f = new File(".", dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); - GitClient g = this.subGit(needleBasename); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); + GitClient g = this.subGit(needleBasename); //FIXME? needle or f? Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); @@ -441,13 +444,15 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Finally check pattern's parent dir arrDirnames.add("."); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) per arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + for (String dirname : arrDirnames) { f = new File(".", dirname); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); - GitClient g = this.subGit(needleBasename); + GitClient g = this.subGit(needleBasename); // needle or f? Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); From ab29c684d1dc70c71664e36cbdfc29c4183fd7f5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:21:17 +0100 Subject: [PATCH 058/155] LegacyCompatibleGitAPIImpl.java: comment reshuffle --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 58b5b59ffc..e7790d1abd 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -551,7 +551,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while - // it means the same thing to known Git platforms. + // it means the same thing to known Git platforms, except local + // dirs on case-sensitive filesystems. // The actual reference repository directory may choose to have // original URL strings added as remotes (in case some are case // sensitive and different). @@ -577,13 +578,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - // TODO: employ git submodules - there we can reliably match - // remote URLs (easily multiple) to particular modules, and - // yet keep separate git index directories per module with - // smaller scopes - much quicker to check out from than one - // huge combined repo. It would also be much more native to - // tools and custom scriptware that can be involved. - // TODO: Config option whether to populate absent reference // repos (If the expanded path does not have git repo data // right now, populate it in the location expanded below) @@ -669,6 +663,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + // Here we employ git submodules - so we can reliably match + // remote URLs (easily multiple) to particular modules, and + // yet keep separate git index directories per module with + // smaller scopes - much quicker to check out from than one + // huge combined repo. It would also be much more native to + // tools and custom scriptware that can be involved. + // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) From 66798fd7dcf409a26c720d7cd21b621fab1419b8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:27:26 +0100 Subject: [PATCH 059/155] LegacyCompatibleGitAPIImpl.java: refactor findParameterizedReferenceRepository() with referenceBaseDir --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e7790d1abd..f3a85dde76 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -558,6 +558,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); + // Drop the trailing keyword to know the root refrepo dirname + String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + @@ -598,7 +601,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems @@ -613,7 +616,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems @@ -636,7 +639,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } @@ -658,7 +661,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } } @@ -696,7 +699,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); @@ -709,7 +712,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } } From 776a249db1bf8398645e5422cc384e05d8c5797e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:45:51 +0100 Subject: [PATCH 060/155] LegacyCompatibleGitAPIImpl.java: trace which sub-git workspaces we do actually look at in getSubmodulesUrls() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f3a85dde76..de427cd71b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -3,6 +3,7 @@ import static java.util.Arrays.copyOfRange; import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; +import hudson.FilePath; import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; @@ -393,6 +394,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); //FIXME? needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); @@ -453,6 +455,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); // needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); From 094de4df709dd0d2c91a7f7713506f5759eca7a6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:35:31 +0100 Subject: [PATCH 061/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to explicitly know referenceBaseDir --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index de427cd71b..8a219be0ee 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -319,12 +319,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * + * @param referenceBaseDir - the reference repository, or container thereof * @param needle - an URL which (or its normalized variant coming from * normalizeGitUrl(url, true)) we want to find: * if it is not null - then stop and return just hits * for it as soon as we have something. */ - public LinkedHashSet getSubmodulesUrls(String needle) { + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -384,15 +385,15 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(".", dirname); + f = new File(referenceBaseDir, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); GitClient g = this.subGit(needleBasename); //FIXME? needle or f? LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -446,14 +447,14 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) per arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(".", dirname); + f = new File(referenceBaseDir, dirname); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); GitClient g = this.subGit(needleBasename); // needle or f? LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -516,9 +517,11 @@ public LinkedHashSet getSubmodulesUrls(String needle) { /** See above. With null, returns all we can find (slower) for the caller * to parse */ - public LinkedHashSet getSubmodulesUrls() { - return getSubmodulesUrls(null); + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { + return getSubmodulesUrls(referenceBaseDir, null); } + /* Do we need a parameter-less variant to look under current work dir + * aka Paths.get("").toAbsolutePath().toString() ?.. */ /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable @@ -682,8 +685,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // to the provided "url". // Note: we pass "url" here, the routine differentiates original - // naming vs. normalization. - LinkedHashSet subEntries = getSubmodulesUrls(url); + // naming vs. normalization while looking for its needle in the + // haystack. + LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From d30de211983004dc49593137e50013c4e2ef5d2e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 00:04:13 +0100 Subject: [PATCH 062/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to look in "f" dirname as the git workspace, not needle --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8a219be0ee..e732b6ec0a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -392,9 +392,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { + String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); - GitClient g = this.subGit(needleBasename); //FIXME? needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.subGit(fAbs); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { @@ -453,9 +454,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(referenceBaseDir, dirname); if (f.exists() && f.isDirectory()) { try { + + String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); - GitClient g = this.subGit(needleBasename); // needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.subGit(fAbs); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { From 7ed0f0f6b2aaf4ca4c04badac5c6555cb3c8f452 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:08:09 +0100 Subject: [PATCH 063/155] GitClient and its implementations: extend API with a newGit(somedir) to inspect other repos with same class as currently used --- src/main/java/hudson/plugins/git/GitAPI.java | 5 +++++ .../jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 6 ++++++ .../org/jenkinsci/plugins/gitclient/GitClient.java | 11 +++++++++++ .../org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 6 ++++++ .../jenkinsci/plugins/gitclient/RemoteGitImpl.java | 5 +++++ 5 files changed, 33 insertions(+) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index ce275fdacc..0d02281fdb 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -156,6 +156,11 @@ public GitClient subGit(String subdir) { return Git.USE_CLI ? super.subGit(subdir) : jgit.subGit(subdir); } + /** {@inheritDoc} */ + public GitClient newGit(String somedir) { + return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); + } + /** {@inheritDoc} */ public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { if (Git.USE_CLI) super.setRemoteUrl(name, url); else jgit.setRemoteUrl(name, url); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index d6dda9cdaa..7c3c0cc92c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -321,6 +321,12 @@ public GitClient subGit(String subdir) { return new CliGitAPIImpl(gitExe, new File(workspace, subdir), listener, environment); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new CliGitAPIImpl(gitExe, new File(somedir), listener, environment); + } + /** * Initialize an empty repository for further git operations. * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 498d73bdc1..afe038b65a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -713,6 +713,17 @@ public interface GitClient { List revList(String ref) throws GitException, InterruptedException; + // --- new instance of same applied class + + /** + * newGit. + * + * @return a IGitAPI implementation to manage another git repository + * with same general settings and implementation as the current one. + * @param somedir a {@link java.lang.String} object. + */ + GitClient newGit(String somedir); + // --- submodules /** diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a46330a4ca..5ff02d0eb6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -214,6 +214,12 @@ public GitClient subGit(String subdir) { return new JGitAPIImpl(new File(workspace, subdir), listener); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new JGitAPIImpl(new File(somedir), listener); + } + /** {@inheritDoc} */ @Override public void setAuthor(String name, String email) throws GitException { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index 68a116010c..5f15406f01 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -617,6 +617,11 @@ public GitClient subGit(String subdir) { return proxy.subGit(subdir); } + /** {@inheritDoc} */ + public GitClient newGit(String somedir) { + return proxy.newGit(somedir); + } + /** * hasGitModules. * From 14cf3451c3aa60dc634c188e41ce41817363ea9c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:15:49 +0100 Subject: [PATCH 064/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to look in referenceGit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e732b6ec0a..7837155757 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -334,9 +334,14 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); + // "this" during a checkout typically represents the job workspace, + // but we want to inspect the reference repository located elsewhere + // with the same implementation as the end-user set up (CliGit/jGit) + GitClient referenceGit = this.newGit(referenceBaseDir); + Boolean isBare = false; try { - isBare = this.isBareRepository(); + isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); } catch (InterruptedException e) { isBare = false; // At least try to look into submodules... } @@ -395,7 +400,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.subGit(fAbs); + GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { @@ -458,7 +463,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.subGit(fAbs); + GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { From 2b61767eb81a805e7a94863b61883c0ee84cc9dc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:16:19 +0100 Subject: [PATCH 065/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to return fAbs string in case of a hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7837155757..f0aebb3602 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -409,7 +409,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } // Cache the finding to avoid the dirname later, if we @@ -472,7 +472,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } // Cache the finding to avoid the dirname later, if we From 1c9a8f570ae41c0cbc9e2c9edc795d7517fb31e0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:16:36 +0100 Subject: [PATCH 066/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to track relative dirname string in case of no hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f0aebb3602..021b6f0186 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -415,7 +415,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -478,7 +478,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From a29b00ce1d35c8c3ba8f30ce273c94b6105ab6e3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 02:41:27 +0100 Subject: [PATCH 067/155] LegacyCompatibleGitAPIImpl.java: clearer trace messages in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 021b6f0186..b0a942198d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -390,16 +390,16 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(referenceBaseDir, dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -453,16 +453,16 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(referenceBaseDir, dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { - String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); From 25bef67503b3892fa5e875548e8f96582a323403 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 02:58:39 +0100 Subject: [PATCH 068/155] LegacyCompatibleGitAPIImpl.java: track referenceBaseDirAbs to compare pathnames (cached hits) more reliably in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0a942198d..65407e5db0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -333,11 +333,12 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); + String referenceBaseDirAbs = Paths.get(referenceBaseDir).toAbsolutePath().toString(); // "this" during a checkout typically represents the job workspace, // but we want to inspect the reference repository located elsewhere // with the same implementation as the end-user set up (CliGit/jGit) - GitClient referenceGit = this.newGit(referenceBaseDir); + GitClient referenceGit = this.newGit(referenceBaseDirAbs); Boolean isBare = false; try { @@ -390,10 +391,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDir, dirname); + f = new File(referenceBaseDirAbs, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { @@ -439,7 +440,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } - File[] directories = new File(".").listFiles(File::isDirectory); + File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { f = new File(dir, ".git"); if (f.exists()) { // May be a file or directory... or symlink to those... @@ -453,10 +454,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDir, dirname); + f = new File(referenceBaseDirAbs, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { From 39dd9b4c9a7709d2b0b5bf4ccb9241f866d77037 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:00:37 +0100 Subject: [PATCH 069/155] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to return fAbs string in case of a hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 65407e5db0..acceb57c78 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -416,7 +416,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -479,7 +479,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From c349b2525e9ccedbe59fe41d331efcdcade342ae Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:00:57 +0100 Subject: [PATCH 070/155] LegacyCompatibleGitAPIImpl.java: comment the ends of long logical blocks --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index acceb57c78..90aa5fdfdb 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -492,7 +492,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (isBare) { return new LinkedHashSet<>(); } + // else... ||true above :) } + // else: look for needle in submodules defined in current non-bare refrepo } else { // ...and if there is no needle? Return info on all git remotes // found under this directory... From 6a7bb18121a501a2514ea55cb9bf33e60c5327a9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:24:45 +0100 Subject: [PATCH 071/155] LegacyCompatibleGitAPIImpl.java: use absolute pathnames "natively" in second phase of needle-searching, and only look for submodule definitions in dirs that ARE git workspaces (avoid looking into parent refrepoBaseDir all the time for simple subdirs) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 90aa5fdfdb..6fb36e2d2c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -399,24 +399,26 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); + if (getObjectsFile(fAbs) != null) { + //LegacyCompatibleGitAPIImpl? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -433,7 +435,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (isBare || true) { arrDirnames.clear(); - // Check subdirs that are git workspaces + // Check subdirs that are git workspaces; note these are absolute pathnames // TODO: Refactor to avoid lookups of dirs that may prove not // needed in the end (aim for less I/Os to find the goal) LinkedHashSet checkedDirs = new LinkedHashSet<>(); @@ -452,34 +454,35 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Finally check pattern's parent dir - arrDirnames.add("."); + arrDirnames.add(referenceBaseDirAbs); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDirAbs, dirname); + f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); + if (getObjectsFile(fAbs) != null) { + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From f4756062cb64bb9c7d8b4cf567431de54a22f7e3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:55:27 +0100 Subject: [PATCH 072/155] LegacyCompatibleGitAPIImpl.java: do not need to check bare-mode dirs for being git repos, twice --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 6fb36e2d2c..0205a1dbce 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -456,7 +456,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add(referenceBaseDirAbs); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -464,25 +464,23 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - if (getObjectsFile(fAbs) != null) { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From 2ddd7df74e33613ae3d5b073bff19ed49fe2f4f4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:57:56 +0100 Subject: [PATCH 073/155] LegacyCompatibleGitAPIImpl.java: check basename-like dirs for being git repos, only (submodules do not host their index and have no "objects" directly inside) --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0205a1dbce..e5039075e2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -399,7 +399,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - if (getObjectsFile(fAbs) != null) { + File fGit = new File(f, ".git"); + if (fGit.exists()) { // file, dir or symlink to those //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); From 721d791b4a5cfd1bc969b7c5278e96890ac60346 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:01:49 +0100 Subject: [PATCH 074/155] LegacyCompatibleGitAPIImpl.java: check the parent refrepo dir for hosting a .git object too --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e5039075e2..7550500e39 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -455,7 +455,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Finally check pattern's parent dir - arrDirnames.add(referenceBaseDirAbs); + if (new File(referenceBaseDirAbs, ".git").exists()) { + arrDirnames.add(referenceBaseDirAbs); + } LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); From 69f5b59eb638ed8026f7380a651d77963a3b0ccc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:36:46 +0100 Subject: [PATCH 075/155] LegacyCompatibleGitAPIImpl.java: trace what getRemoteUrls() gave us --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7550500e39..f168631275 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -406,6 +406,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); @@ -471,6 +472,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); From 9c48cb6685f7dd7e08703be6ff70c05f93189508 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:37:26 +0100 Subject: [PATCH 076/155] LegacyCompatibleGitAPIImpl.java: leave a TODO block for checking submodule definitions in current dir (might recurse that eventually) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f168631275..d467c96cc6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -428,6 +428,39 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } +/* +// TBD: Needs a way to list submodules in given workspace and convert +// that into (relative) subdirs, possibly buried some levels deep, for +// cases where the submodule is defined in parent with the needle URL. +// Maybe merge with current "if isBare" below, to optionally seed +// same arrDirnames with different values and check remotes listed +// in those repos. + // If current repo *is NOT* bare - check its submodules + if (!isBare || true) { + try { + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } +*/ + // If current repo *is* bare (can't have proper submodules) and the // needle is not null, follow up with: // * Maybe also direct child dirs that have a ".git" FS object inside?.. From f465258fb50c7c73fdfa4c274160bed53c769af0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 09:56:24 +0100 Subject: [PATCH 077/155] LegacyCompatibleGitAPIImpl.java: trace mis-processing an assumed git workspace with an exception --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d467c96cc6..356e08e916 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -424,6 +424,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } } @@ -522,6 +523,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } } From f6e756acebeae90111acca4eea690cf751f0774a Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 09:56:41 +0100 Subject: [PATCH 078/155] LegacyCompatibleGitAPIImpl.java: comment about abs dirnames in second phase --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 356e08e916..c49763ad30 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -497,6 +497,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { + // Note that here dirnames deal in absolutes f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { From ce1827e7759ffa96b830525adea471702f599e20 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 10:09:24 +0100 Subject: [PATCH 079/155] LegacyCompatibleGitAPIImpl.java: fix GitClient for abs dirnames in second phase --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c49763ad30..f67891f94a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -504,7 +504,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String try { String fAbs = f.getAbsolutePath().toString(); LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); + GitClient g = this.newGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); From c5e863fed53b6ff7371c3464ccd8bca397f3656d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 10:34:49 +0100 Subject: [PATCH 080/155] LegacyCompatibleGitAPIImpl.java: Log into build console the start and outcome of resolving the parameterized Git reference repository, to explain the clutter from "git" calls --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f67891f94a..a355bd477c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -603,6 +603,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); + // Note: Initially we expect the reference to be a realistic dirname + // with a special suffix to substitute after the logic below, so the + // referencePath for that verbatim funny string should not exist now: if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty() @@ -625,6 +628,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); + // Let users know why there are many "git config --list" lines in their build log: + LOGGER.log(Level.INFO, "Trying to resolve parameterized Git reference repository '" + + reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); + String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, @@ -785,7 +792,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); - } + + LOGGER.log(Level.INFO, "After resolving the parameterized Git reference repository, " + + "decided to use '" + reference + "' directory for URL '" + url + "'"); + } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { // Normalize the URLs with or without .git suffix to From d0bf23c5aeb55ef1931ae4a3d9effa0412194f37 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:25:34 +0100 Subject: [PATCH 081/155] LegacyCompatibleGitAPIImpl.java: Refactor a useless needle to be null, and move declarations of values derived from nun-null needle --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index a355bd477c..338dc57c53 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -347,14 +347,25 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String isBare = false; // At least try to look into submodules... } + // Simplify checks below by stating a useless needle is null + if (needle != null && needle.isEmpty()) { + needle = null; + } + // This is only used and populated if needle is not null + String needleNorm = null; + // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. - if (needle != null && !needle.isEmpty()) { - int sep = needle.lastIndexOf("/"); + if (needle != null) { String needleBasename; + String needleBasenameLC; + String needleNormBasename; + String needleSha; + + int sep = needle.lastIndexOf("/"); if (sep < 0) { needleBasename = needle; } else { @@ -362,9 +373,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } needleBasename = needleBasename.replaceAll(".[Gg][Ii][Tt]$", ""); - String needleNorm = normalizeGitUrl(needle, true); + needleNorm = normalizeGitUrl(needle, true); sep = needleNorm.lastIndexOf("/"); - String needleNormBasename; if (sep < 0) { needleNormBasename = needleNorm; } else { @@ -376,7 +386,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // First we try the caller-provided string casing, then normalized. arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); - String needleBasenameLC = needleBasename.toLowerCase(); + needleBasenameLC = needleBasename.toLowerCase(); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname arrDirnames.add(needleBasenameLC); @@ -387,7 +397,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleNormBasename + ".git"); } - String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); From 0b761da482573e775839ca92ca61d1d27496fea2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:27:11 +0100 Subject: [PATCH 082/155] LegacyCompatibleGitAPIImpl.java: Refactor getSubmodulesUrls() to unify logic needed for both search for a needle or full listing --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 231 +++++++++--------- 1 file changed, 120 insertions(+), 111 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 338dc57c53..72c551d8a0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -438,6 +438,33 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } } + } // if needle, look in basename-like and SHA dirs first + + // Needle or not, the rest of directory walk to collect data is the + // same, so follow a list of whom we want to visit in likely-quickest + // hit order. Note that if needle is null, we walk over everything + // that makes sense to visit, to return info on all git remotes found + // in or under this directory; however if it is not-null, we still + // try to have minimal overhead to complete as soon as we match it. + // TODO: Refactor to avoid lookups of dirs that may prove not needed + // in the end (aim for less I/Os to find the goal)... or is one dir + // listing a small price to pay for maintaining one unified logic? + + // If current dir does have submodules, first dig into submodules, + // when there is no deeper to drill, report remote URLs and step + // back from recursion. This way we have least specific repo last, + // if several have the replica (assuming the first hits are smaller + // scopes). + + // Will build a new shopping list to browse + arrDirnames.clear(); + + // Track where we have looked already; note that values in result[] + // (if any from needle-search above) are absolute pathnames + LinkedHashSet checkedDirs = new LinkedHashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } /* // TBD: Needs a way to list submodules in given workspace and convert @@ -446,135 +473,117 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Maybe merge with current "if isBare" below, to optionally seed // same arrDirnames with different values and check remotes listed // in those repos. - // If current repo *is NOT* bare - check its submodules - if (!isBare || true) { + // If current repo *is NOT* bare - check its submodules + if (!isBare || true) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // getRemoteUrls() => Map +// arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above +// LinkedHashSet checkedDirs = new LinkedHashSet<>(); +// for (String[] resultEntry : result) { +// checkedDirs.add(resultEntry[0]); +// } + + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } +*/ + + // If current repo *is* bare (can't have proper submodules), or if the + // end-users just cloned or linked some more repos into this container, + // follow up with direct child dirs that have a ".git" FS object inside: + + // Check subdirs that are git workspaces; note that values in checkedDirs + // are absolute pathnames + File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); + for (File dir : directories) { + f = new File(dir, ".git"); + if (f.exists()) { // May be a file or directory... or symlink to those... + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } + } + } + + // Finally check pattern's parent dir + // * Look at remote URLs in current dir after the guessed subdirs failed, + // and return then. + if (new File(referenceBaseDirAbs, ".git").exists()) { + arrDirnames.add(referenceBaseDirAbs); + } + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + + ((isBare ? "" : "submodules first, then ")) + + "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + + "' per absolute arrDirnames: " + arrDirnames.toString()); + + for (String dirname : arrDirnames) { + // Note that here dirnames deal in absolutes + f = new File(dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + if (f.exists() && f.isDirectory()) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); + String fAbs = f.getAbsolutePath().toString(); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.newGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + // Cache the finding to return eventually. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } + // TODO: Here is a good spot to recurse this routine into + // a subdir that is a known git workspace, to add its data + // and/or return a found needle. } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } -*/ - - // If current repo *is* bare (can't have proper submodules) and the - // needle is not null, follow up with: - // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdirs failed, - // and return then. - // TODO: Remove "|| true" when other logic fo look for real submodules is complete - if (isBare || true) { - arrDirnames.clear(); - - // Check subdirs that are git workspaces; note these are absolute pathnames - // TODO: Refactor to avoid lookups of dirs that may prove not - // needed in the end (aim for less I/Os to find the goal) - LinkedHashSet checkedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); - } - File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); - for (File dir : directories) { - f = new File(dir, ".git"); - if (f.exists()) { // May be a file or directory... or symlink to those... - String dirname = dir.getPath().replaceAll("/*$", ""); - if (!checkedDirs.contains(dirname)) { - arrDirnames.add(dirname); - } - } - } - - // Finally check pattern's parent dir - if (new File(referenceBaseDirAbs, ".git").exists()) { - arrDirnames.add(referenceBaseDirAbs); - } - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); - - for (String dirname : arrDirnames) { - // Note that here dirnames deal in absolutes - f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); - if (f.exists() && f.isDirectory()) { - try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - } - } catch (Exception e) { - // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); - } - } - } - - // Nothing found, and for bare top-level repo not much to search for - // Or maybe also fall through to let git decide? - if (isBare) { - return new LinkedHashSet<>(); - } - // else... ||true above :) - } - // else: look for needle in submodules defined in current non-bare refrepo - } else { - // ...and if there is no needle? Return info on all git remotes - // found under this directory... - /* if (isBare) { ... } */ - // fall through currently, let git decide if it has submodules here and now - } - - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // getRemoteUrls() => Map - arrDirnames.clear(); + } - // TODO: Check subdirs that are git workspaces, and remove "|| true" above - LinkedHashSet checkedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); + // Nothing found, so if we had a needle - report there are no hits + if (needle != null) { + return new LinkedHashSet<>(); } - // If current dir does have submodules, first dig into submodules, - // when there is no deeper to drill, report remote URLs and step - // back from recursion. This way we have least specific repo last, - // if several have the replica (assuming the first hits are smaller - // scopes). return result; } From 6046f4b1d2cfa1064a83d4f7f4789f3ef3d1e5f7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:44:10 +0100 Subject: [PATCH 083/155] LegacyCompatibleGitAPIImpl.java: Refactor getSubmodulesUrls() to name uriNames pair value a "remoteName" --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 72c551d8a0..e7e047a6d1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -418,18 +418,19 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); return result; } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); } } } catch (Exception e) { @@ -558,16 +559,17 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); return result; } // Cache the finding to return eventually. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); } // TODO: Here is a good spot to recurse this routine into // a subdir that is a known git workspace, to add its data From 3f7b4e4e5ad025cfa33830a6cabc48cfea8a9805 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 14:54:59 +0100 Subject: [PATCH 084/155] LegacyCompatibleGitAPIImpl.java: re-use arrDirnames in getSubmodulesUrls() if it has "prioritized" pathnames, to recurse into them first --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e7e047a6d1..b634402307 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -457,9 +457,6 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // if several have the replica (assuming the first hits are smaller // scopes). - // Will build a new shopping list to browse - arrDirnames.clear(); - // Track where we have looked already; note that values in result[] // (if any from needle-search above) are absolute pathnames LinkedHashSet checkedDirs = new LinkedHashSet<>(); @@ -522,7 +519,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // follow up with direct child dirs that have a ".git" FS object inside: // Check subdirs that are git workspaces; note that values in checkedDirs - // are absolute pathnames + // are absolute pathnames. If we did look for the needle, array already + // starts with some "prioritized" pathnames which we should not directly + // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { f = new File(dir, ".git"); @@ -551,33 +550,41 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { - try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String remoteName = pair.getValue(); // whatever the git workspace config calls it - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { - result = new LinkedHashSet<>(); + if (!checkedDirs.contains(dirname)) { + try { + String fAbs = f.getAbsolutePath().toString(); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.newGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + return result; + } + // Cache the finding to return eventually, for each remote: + // * absolute dirname of a Git workspace + // * original remote URI from that workspace's config + // * normalized remote URI + // * name of the remote from that workspace's config ("origin" etc) result.add(new String[]{fAbs, uri, uriNorm, remoteName}); - return result; } - // Cache the finding to return eventually. - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + } catch (Exception e) { + // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } - // TODO: Here is a good spot to recurse this routine into - // a subdir that is a known git workspace, to add its data - // and/or return a found needle. - } catch (Exception e) { - // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } + + // TODO: Here is a good spot to recurse this routine into + // a subdir that is a known git workspace, to add its data + // and/or return a found needle. + } } From 61a7d0bed352546b79231abfbd788b602a05eba9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 14:55:22 +0100 Subject: [PATCH 085/155] LegacyCompatibleGitAPIImpl.java: comment the plan about submodule search from .gitmodules data --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b634402307..351a4eacb1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -472,7 +472,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // same arrDirnames with different values and check remotes listed // in those repos. // If current repo *is NOT* bare - check its submodules - if (!isBare || true) { + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory + if (!isBare) { try { // For each current workspace (recurse or big loop in same context?): // public GitClient subGit(String subdir) => would this.subGit(...) @@ -481,6 +483,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // faster possible answer (only bother if needle is not null?) // try { getSubmodules("HEAD") ... } => List filtered for // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? // getRemoteUrls() => Map // arrDirnames.clear(); From b91e00b939e299fe4295bf796e9a83946b13e88f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 13:50:55 +0100 Subject: [PATCH 086/155] LegacyCompatibleGitAPIImpl.java: update comments around shorter getSubmodulesUrls() signature --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 351a4eacb1..924dac704e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -603,13 +603,14 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String return result; } - /** See above. With null, returns all we can find (slower) for the caller - * to parse */ + /** See above. With null needle, returns all data we can find under the + * referenceBaseDir tree (can take a while) for the caller to parse */ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { - return getSubmodulesUrls(referenceBaseDir, null); + return getSubmodulesUrls(referenceBaseDir, null, true); } - /* Do we need a parameter-less variant to look under current work dir - * aka Paths.get("").toAbsolutePath().toString() ?.. */ + /* Do we need a completely parameter-less variant to look under current + * work dir aka Paths.get("").toAbsolutePath().toString(), or under "this" + * GitClient workspace ?.. */ /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable From 65684f31b22d57b076a32fe84a26401655c3307f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 17:57:53 +0100 Subject: [PATCH 087/155] LegacyCompatibleGitAPIImpl.java: only track absolute dirname values in getSubmodulesUrls() right away --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 924dac704e..5f2b2cd392 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -384,53 +384,55 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Try with the basename without .git extension, and then with one. // First we try the caller-provided string casing, then normalized. - arrDirnames.add(needleBasename); - arrDirnames.add(needleBasename + ".git"); + // Note that only after this first smaller pass which we hope to + // succeed quickly, we engage in heavier (by I/O and computation) + // investigation of submodules, and then similar loop against any + // remaining direct subdirs that contain a ".git" FS object. + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); needleBasenameLC = needleBasename.toLowerCase(); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname - arrDirnames.add(needleBasenameLC); - arrDirnames.add(needleBasenameLC + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC + ".git"); } if (!needleNormBasename.equals(needleBasenameLC)) { - arrDirnames.add(needleNormBasename); - arrDirnames.add(needleNormBasename + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename + ".git"); } needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); - arrDirnames.add(needleSha); - arrDirnames.add(needleSha + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDirAbs, dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + f = new File(dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { try { - String fAbs = f.getAbsolutePath().toString(); File fGit = new File(f, ".git"); if (fGit.exists()) { // file, dir or symlink to those - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); return result; } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } } catch (Exception e) { @@ -555,12 +557,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { if (!checkedDirs.contains(dirname)) { try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); GitClient g = this.newGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -569,10 +570,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); return result; } // Cache the finding to return eventually, for each remote: @@ -580,7 +581,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // * original remote URI from that workspace's config // * normalized remote URI // * name of the remote from that workspace's config ("origin" etc) - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } catch (Exception e) { // ignore, go to next slide From 0c99c95dc11796433b97a679eb6f7c9a4e972560 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 17:58:09 +0100 Subject: [PATCH 088/155] LegacyCompatibleGitAPIImpl.java: comment the plan for submodule handling --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5f2b2cd392..282d6ca52b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -475,7 +475,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // in those repos. // If current repo *is NOT* bare - check its submodules // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. if (!isBare) { try { // For each current workspace (recurse or big loop in same context?): From 327cb48e56eff2043eca2126b893f368db8f8bef Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 13:50:30 +0100 Subject: [PATCH 089/155] LegacyCompatibleGitAPIImpl.java: finally, do recursion in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 282d6ca52b..c621d54bf4 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -324,8 +324,14 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * normalizeGitUrl(url, true)) we want to find: * if it is not null - then stop and return just hits * for it as soon as we have something. + * @param checkRemotesInReferenceBaseDir - if true (reasonable default for + * external callers), the referenceBaseDir would be added + * to the list of dirs for listing known remotes in search + * for a needle match or for the big listing. Set to false + * when recursing, since this directory was checked already + * as part of parent directory inspection. */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle) { + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -548,7 +554,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (new File(referenceBaseDirAbs, ".git").exists()) { + if (checkRemotesInReferenceBaseDir && new File(referenceBaseDirAbs, ".git").exists()) { arrDirnames.add(referenceBaseDirAbs); } @@ -592,10 +598,20 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } - // TODO: Here is a good spot to recurse this routine into - // a subdir that is a known git workspace, to add its data - // and/or return a found needle. - + // Here is a good spot to recurse this routine into a + // subdir that is already a a known git workspace, to + // add its data to list and/or return a found needle. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); + if (!subEntries.isEmpty()) { + if (needle != null) { + // We found nothing... until now! Bubble it up! + LOGGER.log(Level.FINE, "getSubmodulesUrls(): got a needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + return subEntries; + } + result.addAll(subEntries); + } } } @@ -787,7 +803,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass "url" here, the routine differentiates original // naming vs. normalization while looking for its needle in the // haystack. - LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url); + LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From 45d268cc7b3424f5acacb05605d2ff41cd049024 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:04:01 +0100 Subject: [PATCH 090/155] LegacyCompatibleGitAPIImpl.java: check not only for ".git" but also "objects" in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c621d54bf4..f06d42aee7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -393,7 +393,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Note that only after this first smaller pass which we hope to // succeed quickly, we engage in heavier (by I/O and computation) // investigation of submodules, and then similar loop against any - // remaining direct subdirs that contain a ".git" FS object. + // remaining direct subdirs that contain a ".git" (or "objects") + // FS object. arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); needleBasenameLC = needleBasename.toLowerCase(); @@ -418,8 +419,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { try { - File fGit = new File(f, ".git"); - if (fGit.exists()) { // file, dir or symlink to those + File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(f, "objects"); // bare + if (fGit.exists() || fObj.exists()) { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); @@ -534,7 +536,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, - // follow up with direct child dirs that have a ".git" FS object inside: + // follow up with direct child dirs that have a ".git" (or "objects") + // FS object inside: // Check subdirs that are git workspaces; note that values in checkedDirs // are absolute pathnames. If we did look for the needle, array already @@ -542,8 +545,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { - f = new File(dir, ".git"); - if (f.exists()) { // May be a file or directory... or symlink to those... + File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(f, "objects"); // bare + if (fGit.exists() || fObj.exists()) { String dirname = dir.getPath().replaceAll("/*$", ""); if (!checkedDirs.contains(dirname)) { arrDirnames.add(dirname); @@ -554,8 +558,12 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (checkRemotesInReferenceBaseDir && new File(referenceBaseDirAbs, ".git").exists()) { - arrDirnames.add(referenceBaseDirAbs); + if (checkRemotesInReferenceBaseDir) { + File fGit = new File(referenceBaseDirAbs, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(referenceBaseDirAbs, "objects"); // bare + if (fGit.exists() || fObj.exists()) { + arrDirnames.add(referenceBaseDirAbs); + } } LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + @@ -568,6 +576,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { + // No checks for ".git" or "objects" this time, already checked above if (!checkedDirs.contains(dirname)) { try { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); From 0413d95fd4f88aaf221f292232621294f3eb5332 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:21:47 +0100 Subject: [PATCH 091/155] LegacyCompatibleGitAPIImpl.java: for GIT_SUBMODULES mode search, if there were no hits, suggest the basename(URL) subdir (friendlier for co-hosting forks) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f06d42aee7..72e7af1565 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -803,15 +803,20 @@ public File findParameterizedReferenceRepository(String reference, String url) { // smaller scopes - much quicker to check out from than one // huge combined repo. It would also be much more native to // tools and custom scriptware that can be involved. + // Beside git-submodule parsing (that only points to one URL + // at a time) his also covers a search for subdirectories + // that host a git repository whose remotes match the URL, + // to handle co-hosting of several remotes (different URLs + // to same repository, e.g. SSH and HTTPS; mirrors; forks). // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) // to the provided "url". - // Note: we pass "url" here, the routine differentiates original - // naming vs. normalization while looking for its needle in the - // haystack. + // Note: we pass the unmodified "url" value here, the routine + // differentiates original spelling vs. normalization while + // looking for its needle in the haystack. LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted @@ -836,15 +841,29 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new - // directory name to host the submodule (same rules as SHA), + // directory name to host the submodule (same rules as for + // the refrepo forks' co-hosting friendly basename search), // and the fallback mode would return the main directory. + int sep = url.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = url; + } else { + needleBasename = url.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + needleBasename); } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - // chop it off, use main directory - referenceExpanded = referenceBaseDir; + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", + needleBasename); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } } } } From fad3727ab577e8b4a5bd97fffb79d9c0cebcf6c9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:58:42 +0100 Subject: [PATCH 092/155] LegacyCompatibleGitAPIImpl.java: implement a use-case of ".git" file with "gitdir" entry, for getObjectsFile() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 72e7af1565..0a1b7f439d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -19,7 +19,9 @@ import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; @@ -184,6 +186,9 @@ public File getObjectsFile(String reference) { } public File getObjectsFile(File reference) { + // reference pathname can either point to a "normal" workspace + // checkout or a bare repository + if (reference == null) { return reference; } @@ -194,14 +199,63 @@ public File getObjectsFile(File reference) { if (!reference.isDirectory()) return null; - // reference pathname can either point to a normal or a base repository - File objects = new File(reference, ".git/objects"); - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved + File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those + //File fObj = new File(f, "objects"); // bare + File objects = null; + + if (fGit.exists()) { + if (fGit.isDirectory()) { + objects = new File(fGit, "objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } + } else { + // If ".git" FS object exists and is a not-empty file (and + // is not a dir), then its content may point to some other + // filesystem location for the Git-internal data. + // For example, a checked-out submodule workspace can point + // to the index and other metadata stored in its "parent" + // repository's directory: + // "gitdir: ../.git/modules/childRepoName" + BufferedReader reader = null; + try { + String line; + reader = new BufferedReader(new FileReader(fGit)); + while ((line = reader.readLine()) != null) + { + String[] parts = line.split(":", 2); + if (parts.length >= 2) + { + String key = parts[0].trim(); + String value = parts[1].trim(); + if (key.equals("gitdir")) { + objects = new File(reference, value); + LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', found reference to objects " + + "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + break; + } + LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + } + } + if (objects == null) { + LOGGER.log(Level.WARNING, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + objects = null; + } + try { + if (reader != null) { + reader.close(); + } + } catch (IOException e) {} + } } - if (!objects.isDirectory()) { - // reference path is bare repo + if (objects == null || !objects.isDirectory()) { + // reference path is bare repo (no .git inside) or failed interpreting ".git" contents objects = new File(reference, "objects"); if (objects == null) { return objects; // Some Java error, could not make object from the paths involved From d37fe49ca7d9d20be918e46d4fa140005cd61d67 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 19:54:22 +0100 Subject: [PATCH 093/155] LegacyCompatibleGitAPIImpl.java: refactor getSubmodulesUrls() to use common getObjectsFile() instead of direct FS object queries --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0a1b7f439d..37c26fc193 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -471,31 +471,27 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String dirname : arrDirnames) { f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); - if (f.exists() && f.isDirectory()) { + if (getObjectsFile(f) != null) { try { - File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(f, "objects"); // bare - if (fGit.exists() || fObj.exists()) { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String remoteName = pair.getValue(); // whatever the git workspace config calls it - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } catch (Exception e) { // ignore, go to next slide @@ -599,9 +595,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { - File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(f, "objects"); // bare - if (fGit.exists() || fObj.exists()) { + if (getObjectsFile(dir) != null) { String dirname = dir.getPath().replaceAll("/*$", ""); if (!checkedDirs.contains(dirname)) { arrDirnames.add(dirname); @@ -613,9 +607,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. if (checkRemotesInReferenceBaseDir) { - File fGit = new File(referenceBaseDirAbs, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(referenceBaseDirAbs, "objects"); // bare - if (fGit.exists() || fObj.exists()) { + if (getObjectsFile(referenceBaseDirAbs) != null) { arrDirnames.add(referenceBaseDirAbs); } } @@ -631,6 +623,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above + // by getObjectsFile(). Probably should not check exists/dir either, + // but better be on the safe side :) if (!checkedDirs.contains(dirname)) { try { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); @@ -662,7 +656,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Here is a good spot to recurse this routine into a - // subdir that is already a a known git workspace, to + // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); From 6afacdab311161e83b62ff88038f6fc9f4138736 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 19:46:25 +0100 Subject: [PATCH 094/155] LegacyCompatibleGitAPIImpl.java: refactor getSubmodulesUrls() to return both a Boolean whether it had an exact match, and the result Set --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 37c26fc193..67c19b60d6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashSet; @@ -359,10 +360,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { return urlNormalized; } - /** Find referenced URLs in this repo and its submodules, recursively. - * Current primary use is for parameterized refrepo/${GIT_SUBMODULES} + /** Find referenced URLs in this repo and its submodules (or other + * subdirs with git repos), recursively. Current primary use is for + * parameterized refrepo/${GIT_SUBMODULES} handling. * - * @return a Set of (unique) String arrays, representing: + * @return an AbstractMap.SimpleEntry, containing a Boolean to denote + * an exact match (or lack thereof) for the needle (if searched for), + * and a Set of (unique) String arrays, representing: * [0] directory of nested submodule (relative to current workspace root) * The current workspace would be listed as directory "" and consumers * should check these entries last if they care for most-specific hits @@ -373,6 +377,17 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * + * If the returned SimpleEntry has the Boolean flag as False but also + * a Set which is not empty, and a search for "needle" was requested, + * then that Set lists some not-exact matches for existing sub-dirs + * with repositories that seem likely to be close hits (e.g. remotes + * there *probably* point to other URLs of same repo as the needle, + * or its forks - so these directories are more likely than others to + * contain the reference commits needed for the faster git checkouts). + * + * For a search with needle==null, the Boolean flag would be False too, + * and the Set would just detail all found sub-repositories. + * * @param referenceBaseDir - the reference repository, or container thereof * @param needle - an URL which (or its normalized variant coming from * normalizeGitUrl(url, true)) we want to find: @@ -385,7 +400,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * when recursing, since this directory was checked already * as part of parent directory inspection. */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { + public SimpleEntry> getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -486,7 +501,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; + return new SimpleEntry<>(true, result); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop @@ -640,7 +655,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; + return new SimpleEntry<>(true, result); } // Cache the finding to return eventually, for each remote: // * absolute dirname of a Git workspace @@ -659,30 +674,40 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); + SimpleEntry > subEntriesRet = getSubmodulesUrls(dirname, needle, false); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { - if (needle != null) { + if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! - LOGGER.log(Level.FINE, "getSubmodulesUrls(): got a needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); - return subEntries; + LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + return subEntriesRet; } result.addAll(subEntries); } } } - // Nothing found, so if we had a needle - report there are no hits + // Nothing found, if we had a needle - so report there are no hits if (needle != null) { - return new LinkedHashSet<>(); + // TODO: Handle suggestions (not-exact matches) if something from + // results looks like it is related to the needle. +/* + if (checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once? + } +*/ + return new SimpleEntry<>(false, new LinkedHashSet<>()); } - return result; + // Did not look for anything in particular + return new SimpleEntry<>(false, result); } /** See above. With null needle, returns all data we can find under the * referenceBaseDir tree (can take a while) for the caller to parse */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { + public SimpleEntry> getSubmodulesUrls(String referenceBaseDir) { return getSubmodulesUrls(referenceBaseDir, null, true); } /* Do we need a completely parameter-less variant to look under current @@ -865,17 +890,21 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass the unmodified "url" value here, the routine // differentiates original spelling vs. normalization while // looking for its needle in the haystack. - LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); + SimpleEntry > subEntriesRet = getSubmodulesUrls(referenceBaseDir, url, true); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet subEntries = subEntriesRet.getValue(); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new // reference repo would have to be made (and none exists). // If several entries are present after all, iterate until // first existing hit and return the first entry otherwise. - for (String[] subEntry : subEntries) { - if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { - referenceExpanded = subEntry[0]; - break; + if (!subEntriesExactMatched) { // else look at first entry below + for (String[] subEntry : subEntries) { + if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { + referenceExpanded = subEntry[0]; + break; + } } } if (referenceExpanded == null) { From b278523e4127d916eb5ccd7441af4276444e9f6f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 13:30:46 +0100 Subject: [PATCH 095/155] LegacyCompatibleGitAPIImpl.java: add logging for getObjectsFile() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 67c19b60d6..b0737c1790 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -210,6 +210,8 @@ public File getObjectsFile(File reference) { if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } + LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -218,6 +220,8 @@ public File getObjectsFile(File reference) { // to the index and other metadata stored in its "parent" // repository's directory: // "gitdir: ../.git/modules/childRepoName" + LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; @@ -253,6 +257,9 @@ public File getObjectsFile(File reference) { } } catch (IOException e) {} } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + + fGit.getAbsolutePath().toString() + "'"); } if (objects == null || !objects.isDirectory()) { @@ -261,6 +268,19 @@ public File getObjectsFile(File reference) { if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } + // This clause below is redundant for production, but useful for troubleshooting + if (objects.exists()) { + if (objects.isDirectory()) { + LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is a directory"); + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is NOT a directory"); + } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + + objects.getAbsolutePath().toString() + "'"); + } } if (!objects.isDirectory()) From 446844355ee5072bc19303038e53db13b2a3b476 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 14:33:39 +0100 Subject: [PATCH 096/155] LegacyCompatibleGitAPIImpl.java: in getObjectsFile(), comment why it does not return "object" dir File named with getCanonicalPath() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0737c1790..7ef63e1a00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -238,6 +238,14 @@ public File getObjectsFile(File reference) { LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath().toString() + "', found reference to objects " + "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + // Note: we don't use getCanonicalPath() here to avoid further filesystem + // access and possible exceptions (the getAbsolutePath() is about string + // processing), but callers might benefit from canonicising and ensuring + // unique pathnames (for equality checks etc.) with relative components + // and symlinks resolved. + // On another hand, keeping the absolute paths, possibly, relative to a + // parent directory as prefix, allows callers to match/compare such parent + // prefixes for the contexts the callers would define for themselves. break; } LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + From 0edcc0c2c2a30977eca15aec2506caa2c73c3f38 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 14:46:40 +0100 Subject: [PATCH 097/155] JGitAPIImpl.java / CliGitAPIImpl.java: inform the build console log reader why they see lots of "git rev-parse --is-bare-repository" + "git config --local --list" messages --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +++++ .../java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 7c3c0cc92c..34e97fb5f1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -767,6 +767,11 @@ public void execute() throws GitException, InterruptedException { } if (reference != null && !reference.isEmpty()) { + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener + listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + + "it may take a few git queries logged below to resolve it into a particular directory name"); + } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 5ff02d0eb6..c407f7b1ec 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1456,6 +1456,11 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener + listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + + "it may take a few git queries logged below to resolve it into a particular directory name"); + } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); From 0767c2a3ecb67b2c27ef213abe548abfa483f2cc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 15:33:26 +0100 Subject: [PATCH 098/155] LegacyCompatibleGitAPIImpl.java: in getObjectsFile() method comment, clarify when "objects" may be not under "reference" --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7ef63e1a00..1cd4a225cc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -174,6 +174,8 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, /** For referenced directory check if it is a full or bare git repo * and return the File object for its "objects" sub-directory. + * (Note that for submodules and other cases with externalized Git + * metadata, the "objects" directory may be NOT under "reference"). * If there is nothing to find, or inputs are bad, returns null. * The idea is that checking for null allows to rule out non-git * paths, while a not-null return value is instantly usable by From 03b30564402acc42465b750a4d3b2ed5e9fd22c6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 15:36:26 +0100 Subject: [PATCH 099/155] LegacyCompatibleGitAPIImpl.java: little clean-up in getObjectsFile() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 1cd4a225cc..2f74e8c305 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -203,7 +203,6 @@ public File getObjectsFile(File reference) { return null; File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those - //File fObj = new File(f, "objects"); // bare File objects = null; if (fGit.exists()) { @@ -273,8 +272,9 @@ public File getObjectsFile(File reference) { } if (objects == null || !objects.isDirectory()) { - // reference path is bare repo (no .git inside) or failed interpreting ".git" contents - objects = new File(reference, "objects"); + // either reference path is bare repo (no ".git" inside), + // or we have failed interpreting ".git" contents above + objects = new File(reference, "objects"); // bare if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } From fd88900a6946537bb004a358a7305e5f7abea181 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 16:57:30 +0100 Subject: [PATCH 100/155] LegacyCompatibleGitAPIImpl.java: referenceBaseDirAbs in getSubmodulesUrls() should be canonical (thus unique), if possible --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2f74e8c305..283aa3ffcf 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -438,7 +438,19 @@ public SimpleEntry> getSubmodulesUrls(String re File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); - String referenceBaseDirAbs = Paths.get(referenceBaseDir).toAbsolutePath().toString(); + + // Note: an absolute path is not necessarily the canonical one + // We want to hit same dirs only once, so canonicize paths below + String referenceBaseDirAbs; + try { + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath().toString(); + } catch (IOException e) { + // Access error while dereferencing some parent?.. + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); + LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to canonicize '" + + referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + e.toString()); + //return new SimpleEntry<>(false, result); + } // "this" during a checkout typically represents the job workspace, // but we want to inspect the reference repository located elsewhere From d72999be39aac7a14f01ba4306c8b43d84ad87c9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 16:58:09 +0100 Subject: [PATCH 101/155] LegacyCompatibleGitAPIImpl.java: suggest a selection of results[] *similar* to needle in getSubmodulesUrls() if no exact match happened --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 96 +++++++++++++++++-- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 283aa3ffcf..583875937a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -471,17 +471,19 @@ public SimpleEntry> getSubmodulesUrls(String re // This is only used and populated if needle is not null String needleNorm = null; + // This is only used and populated if needle is not null, and + // can be used in the end to filter not-exact match suggestions + String needleBasename = null; + String needleBasenameLC = null; + String needleNormBasename = null; + String needleSha = null; + // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. if (needle != null) { - String needleBasename; - String needleBasenameLC; - String needleNormBasename; - String needleSha; - int sep = needle.lastIndexOf("/"); if (sep < 0) { needleBasename = needle; @@ -726,21 +728,97 @@ public SimpleEntry> getSubmodulesUrls(String re LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } + // ...else collect results to inspect and/or propagate later result.addAll(subEntries); } } } // Nothing found, if we had a needle - so report there are no hits + // If we did not have a needle, we did not search for it - return + // below whatever result we have, then. if (needle != null) { - // TODO: Handle suggestions (not-exact matches) if something from + if (result.size() == 0) { + // Completely nothing git-like found here, return quickly + return new SimpleEntry<>(false, result); + } + + // Handle suggestions (not-exact matches) if something from // results looks like it is related to the needle. + LinkedHashSet resultFiltered = new LinkedHashSet<>(); + /* - if (checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once? + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); } */ - return new SimpleEntry<>(false, new LinkedHashSet<>()); + + // Separate lists by suggestion priority: + // 1) URI basename similarity + // 2) Directory basename similarity + LinkedHashSet resultFiltered1 = new LinkedHashSet<>(); + LinkedHashSet resultFiltered2 = new LinkedHashSet<>(); + + LinkedHashSet suggestedDirs = new LinkedHashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + + for (String[] subEntry : result) { + // Iterating to filter suggestions in order of original + // directory-walk prioritization under current reference + String dirName = subEntry[0]; + String uri = subEntry[1]; + String uriNorm = subEntry[2]; + String remoteName = subEntry[3]; + Integer sep; + String uriNormBasename; + String dirBasename; + + // Match basename of needle vs. a remote tracked by an + // existing git directory (automation-ready normalized URL) + sep = uriNorm.lastIndexOf("/"); + if (sep < 0) { + uriNormBasename = uriNorm; + } else { + uriNormBasename = uriNorm.substring(sep + 1); + } + uriNormBasename = uriNormBasename.replaceAll(".git$", ""); + + if (uriNormBasename.equals(needleNormBasename)) { + resultFiltered1.add(subEntry); + } + + if (!suggestedDirs.contains(dirName)) { + // Here just match basename of needle vs. an existing + // sub-git directory base name + suggestedDirs.add(dirName); + + sep = dirName.lastIndexOf("/"); + if (sep < 0) { + dirBasename = dirName; + } else { + dirBasename = dirName.substring(sep + 1); + } + dirBasename = dirBasename.replaceAll(".git$", ""); + + if (dirBasename.equals(needleNormBasename)) { + resultFiltered2.add(subEntry); + } + } + } + + // Concatenate suggestions in order of priority. + // Hopefully the Set should deduplicate entries + // if something matched twice :) + resultFiltered.addAll(resultFiltered1); // URLs + resultFiltered.addAll(resultFiltered2); // Dirnames + + // Note: flag is false since matches (if any) are + // not exactly for the Git URL requested by caller + return new SimpleEntry<>(false, resultFiltered); } // Did not look for anything in particular From f1f478a0009bd18d2c1bfe5eef9ff62ee772c4ed Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 18 Jan 2021 02:05:46 +0100 Subject: [PATCH 102/155] LegacyCompatibleGitAPIImpl.java: mark helpers as public static --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 583875937a..10af938135 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -181,14 +181,14 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, * paths, while a not-null return value is instantly usable by * some code which plays with git under its hood. */ - public File getObjectsFile(String reference) { + public static File getObjectsFile(String reference) { if (reference == null || reference.isEmpty()) { return null; } return getObjectsFile(new File(reference)); } - public File getObjectsFile(File reference) { + public static File getObjectsFile(File reference) { // reference pathname can either point to a "normal" workspace // checkout or a bare repository @@ -306,14 +306,14 @@ public File getObjectsFile(File reference) { * * @param reference Pathname (maybe with magic suffix) to reference repo */ - public Boolean isParameterizedReferenceRepository(File reference) { + public static Boolean isParameterizedReferenceRepository(File reference) { if (reference == null) { return false; } return isParameterizedReferenceRepository(reference.getPath()); } - public Boolean isParameterizedReferenceRepository(String reference) { + public static Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; } From 91ed630ff9ad751c4ba98b8691a0ae353f1976d4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 18 Jan 2021 15:19:38 +0100 Subject: [PATCH 103/155] LegacyCompatibleGitAPIImpl.java: check if referenceBaseDir is usable (exists and is a dir) before trying a useless and failure-prone search in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 10af938135..267342b10a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -868,6 +868,19 @@ public File findParameterizedReferenceRepository(String reference, String url) { isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty() ) { + // Drop the trailing keyword to know the root refrepo dirname + String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); + + File referenceBaseDirFile = new File(referenceBaseDir); + if (!referenceBaseDirFile.exists()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); + return null; + } + if (!referenceBaseDirFile.isDirectory()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); + return null; + } + // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while @@ -878,9 +891,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); - // Drop the trailing keyword to know the root refrepo dirname - String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); - // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + From de98cd13daf9aaa983d645d9545e57d4d206cb57 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:04:06 +0100 Subject: [PATCH 104/155] GitClientCloneTest.java: reword original test for GIT_URL into checking GIT_URL_BASENAME --- .../plugins/gitclient/GitClientCloneTest.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 5bcb91bf80..e9ae5bcab5 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -240,8 +240,8 @@ public void test_clone_reference() throws Exception, IOException, InterruptedExc } @Test - public void test_clone_reference_parameterized() throws Exception, IOException, InterruptedException { - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL}").execute(); + public void test_clone_reference_parameterized_basename() throws Exception, IOException, InterruptedException { + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written @@ -251,13 +251,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, handler.containsMessageSubstring(" replaced with: "), is(true)); // TODO: Actually construct the local filesystem path to match - // the literally substituted URL. Note this may not work for - // filesystems and operating systems that constrain character - // ranges usable for filenames (Windows at least) so failure - // to create the paths is an option to handle in test. - // Be sure to clean away this path at end of test, so that the - // test_clone_reference_parameterized_fallback() below is not - // confused - it expects this location to not exist. + // the last pathname component from the URL (plus/minus ".git" + // extension). Be sure to clean away this path at end of test, + // so that the test_clone_reference_parameterized_basename_fallback() + // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); @@ -267,10 +264,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, } @Test - public void test_clone_reference_parameterized_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_basename_fallback() throws Exception, IOException, InterruptedException { // TODO: Currently we do not make paths which would invalidate // this test, but note the test above might do just that later. - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_FALLBACK}").execute(); + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written @@ -281,9 +278,9 @@ public void test_clone_reference_parameterized_fallback() throws Exception, IOEx is(true)); // With fallback mode, and nonexistent parameterized reference // repository, and a usable repository in the common path (what - // remains if the parameterizing suffix is just discarded), - // this common path should be used. So it should overall behave - // same as the non-parameterized test_clone_reference() above. + // remains if the parameterizing suffix is just discarded), this + // common path should be used. So it should overall behave same + // as the non-parameterized test_clone_reference_basename() above. assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); assertAlternateFilePointsToLocalMirror(); assertBranchesExist(testGitClient.getBranches(), "master"); From aa9eaadadc85c6d4f7f88f5bf97cc486fb3a3d28 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:18:26 +0100 Subject: [PATCH 105/155] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): move generic comments about the design from GIT_URL handling to head of function --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 267342b10a..34d918f41b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -861,6 +861,36 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); + // For mass-configured jobs, like Organization Folders, which inherit + // a refrepo setting (String) into generated MultiBranch Pipelines and + // leaf jobs made for each repo branch and PR, the code below allows + // us to support parameterized paths, with one string leading to many + // reference repositories fanned out under a common location. + // This also works for legacy jobs using a Git SCM. + + // TODO: Consider a config option whether to populate absent reference + // repos (If the expanded path does not have git repo data right now, + // should we populate it into the location expanded by logic below), + // or update existing ones before pulling commits, and how to achieve + // that. Currently this is something that comments elsewhere in the + // git-client-plugin and/or articles on reference repository setup + // considered to be explicitly out of scope of the plugin. + // Note that this pre-population (or update) is done by caller with + // their implementation of git and site-specific connectivity and + // storage desigh, e.g. in case of a build farm it is more likely + // to be a shared path from a common storage server only readable to + // Jenkins and its agents, so write-operations would be done by helper + // scripts that log into the shared storage server to populate or + // update the reference repositories. Note that users may also + // want to run their own scripts to "populate" reference repos + // as symlinks to existing other repos, to support combined + // repo setup for different URLs pointing to same upstream, + // or storing multiple closely related forks together. + // This feature was developed along with a shell script to manage + // reference repositories, both in original combined-monolith layout, + // and in the subdirectory fanout compatible with plugin code below: + // https://github.com/jimklimov/git-scripts/blob/master/register-git-cache.sh + // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: @@ -902,8 +932,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { - // For mass-configured jobs, like Organization Folders, - // allow to support parameterized paths to many refrepos. // End-users can set up webs of symlinks to same repos // known by different URLs (and/or including their forks // also cached in same index). Luckily all URL chars are @@ -915,21 +943,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - // TODO: Config option whether to populate absent reference - // repos (If the expanded path does not have git repo data - // right now, populate it in the location expanded below) - // or update existing ones before pulling commits, and how - // to achieve that. Note that this is done by caller with - // their implementation of git, or in case of a build farm it - // is more likely to be a shared path only readable to Jenkins - // and its agents, so write-operations would be done by helper - // scripts that log into the shared storage server to populate - // or update reference repositories. Note that users may also - // want to run their own scripts to "populate" reference repos - // as symlinks to existing other repos, to support combined - // repo setup for different URLs pointing to same upstream, - // or storing multiple closely related forks together. - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); From 82a43eb918d7f07c0edc1101bed2cc679909c6b9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:19:54 +0100 Subject: [PATCH 106/155] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): drop original-experimental support for verbatim GIT_URL as the sub-pathname, due to its little portability --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 34d918f41b..f4452aa1b9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -318,14 +318,6 @@ public static Boolean isParameterizedReferenceRepository(String reference) { return false; } - if (reference.endsWith("/${GIT_URL}")) { - return true; - } - - if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - return true; - } - if (reference.endsWith("/${GIT_URL_SHA256}")) { return true; } @@ -931,26 +923,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); String referenceExpanded = null; - if (reference.endsWith("/${GIT_URL}")) { - // End-users can set up webs of symlinks to same repos - // known by different URLs (and/or including their forks - // also cached in same index). Luckily all URL chars are - // valid parts of path name... in Unix... Maybe parse or - // escape chars for URLs=>paths with Windows in mind? - // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions - // Further ideas: beside "GIT_URL" other meta variables - // can be introduced, e.g. to escape non-ascii chars for - // portability? Support base64, SHA or MD5 hashes of URLs - // as pathnames? Normalize first (lowercase, .git, ...)? - - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); - } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = referenceBaseDir; - } - } else if (reference.endsWith("/${GIT_URL_SHA256}")) { + if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); From 23745a8719fbb95cfccab30c61b462e91b2e763f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:24:55 +0100 Subject: [PATCH 107/155] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: listFiles() can return null --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f4452aa1b9..423ebc0223 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -645,11 +645,15 @@ public SimpleEntry> getSubmodulesUrls(String re // starts with some "prioritized" pathnames which we should not directly // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); - for (File dir : directories) { - if (getObjectsFile(dir) != null) { - String dirname = dir.getPath().replaceAll("/*$", ""); - if (!checkedDirs.contains(dirname)) { - arrDirnames.add(dirname); + if (directories != null) { + // listFiles() "...returns null if this abstract pathname + // does not denote a directory, or if an I/O error occurs" + for (File dir : directories) { + if (getObjectsFile(dir) != null) { + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } } } } From f51e60f4554236031270357020d1b5fd7c03e9f0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:32:12 +0100 Subject: [PATCH 108/155] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: getSubmodulesUrls(): catch generic Throwable, not just Exception, while digging into submodules --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 423ebc0223..98fa23319b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -544,9 +544,9 @@ public SimpleEntry> getSubmodulesUrls(String re // which by construct looks at different dirs so far. result.add(new String[]{dirname, uri, uriNorm, remoteName}); } - } catch (Exception e) { + } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); } } } @@ -704,9 +704,9 @@ public SimpleEntry> getSubmodulesUrls(String re // * name of the remote from that workspace's config ("origin" etc) result.add(new String[]{dirname, uri, uriNorm, remoteName}); } - } catch (Exception e) { + } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); } } From a5fdbd3783e19cbf87c2e18cae051fc975e3f429 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:34:36 +0100 Subject: [PATCH 109/155] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: have non-zero handling for caught IOExpection --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 98fa23319b..ed38cd9752 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -264,7 +264,9 @@ public static File getObjectsFile(File reference) { if (reader != null) { reader.close(); } - } catch (IOException e) {} + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + } } } else { LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + From 4341cd2831dfa1173c47953e86e5bed23a232206 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 21:03:58 +0100 Subject: [PATCH 110/155] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: "new File()" should never return null --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ed38cd9752..0c20978b25 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -207,10 +207,12 @@ public static File getObjectsFile(File reference) { if (fGit.exists()) { if (fGit.isDirectory()) { - objects = new File(fGit, "objects"); - if (objects == null) { + objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below +/* + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null return objects; // Some Java error, could not make object from the paths involved } +*/ LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { @@ -277,9 +279,11 @@ public static File getObjectsFile(File reference) { // either reference path is bare repo (no ".git" inside), // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare +/* if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } +*/ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { if (objects.isDirectory()) { @@ -295,6 +299,9 @@ public static File getObjectsFile(File reference) { } } + if (!objects.exists()) + return null; + if (!objects.isDirectory()) return null; From 4cd45c82d688642443838ee2ac78732d0dcf245e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 21:10:21 +0100 Subject: [PATCH 111/155] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: use a file reader with specified encoding --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0c20978b25..eac141bd18 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -21,8 +21,9 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; @@ -228,7 +229,7 @@ public static File getObjectsFile(File reference) { BufferedReader reader = null; try { String line; - reader = new BufferedReader(new FileReader(fGit)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); From fd293d78c71c67f4e377404c60287214010cf2bc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 22 Feb 2021 16:17:49 +0100 Subject: [PATCH 112/155] LegacyCompatibleGitAPIImpl.java: normalizeUrl() failed with full paths (tried as relative) on Windows --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index eac141bd18..d4eb8895a1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -369,7 +369,21 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { if (checkLocalPaths) { File urlPath = new File(url); if (urlPath.exists()) { - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + try { + // Check if the string in urlNormalized is a valid + // relative path (subdir) in current working directory + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString(); + } catch (java.nio.file.InvalidPathException ipe1) { + // e.g. Illegal char <:> at index 30: C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone + try { + // Re-check in another manner + urlNormalized = "file://" + Paths.get( Paths.get("", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + } catch (java.nio.file.InvalidPathException ipe2) { + // Finally, fall back to checking the originally + // fully-qualified path + urlNormalized = "file://" + Paths.get( Paths.get("/", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + } + } } else { // Also not a subdirectory of current dir without "./" prefix... urlNormalized = "ssh://" + urlNormalized; From c45cedd75c07a5e563be8d14816c51ff32d2ebc1 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 22 Feb 2021 16:56:41 +0100 Subject: [PATCH 113/155] LegacyCompatibleGitAPIImpl.java: normalizeUrl(): @SuppressFBWarnings for absolute "/" path in some lines --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d4eb8895a1..d28fec590a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -359,6 +359,9 @@ public static Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ + @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Path operations below intentionally use absolute '/' in some cases" + ) public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { From be57864f0b937810ee055bc94868dd37f5c17fde Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 26 Jan 2022 21:58:18 +0100 Subject: [PATCH 114/155] CliGitAPIImpl.java: launchCommandIn(): report workDir when throwing exception --- .../org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 34e97fb5f1..dd1ded53d7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -2553,7 +2553,12 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e } if (status != 0) { - throw new GitException("Command \"" + command + "\" returned status code " + status + ":\nstdout: " + stdout + "\nstderr: "+ stderr); + if (workDir == null) + workDir = java.nio.file.Paths.get(".").toAbsolutePath().normalize().toFile(); + throw new GitException("Command \"" + command + + "\" executed in workdir \"" + workDir.toString() + + "\" returned status code " + status + + ":\nstdout: " + stdout + "\nstderr: " + stderr); } return stdout; From 8f843d452bbcfb87350a1b26f23e6e676c48a5ba Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 26 Jan 2022 23:28:53 +0100 Subject: [PATCH 115/155] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls(): do not fail checkout due to not-initialized referenceBaseDir (no git repo there) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d28fec590a..453346d12d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -479,8 +479,25 @@ public SimpleEntry> getSubmodulesUrls(String re Boolean isBare = false; try { isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); - } catch (InterruptedException e) { - isBare = false; // At least try to look into submodules... + } catch (InterruptedException | GitException e) { + // Proposed base directory whose subdirs contain refrepos might + // itself be not a repo. Shouldn't be per reference scripts, but... + if (e.toString().contains("GIT_DISCOVERY_ACROSS_FILESYSTEM")) { + // Note the message may be localized, envvar name should not be: + // stderr: fatal: not a git repository (or any parent up to mount point /some/path) + // Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). + // As far as the logic below is currently concerned, we do not + // look for submodules directly in a bare repo. + isBare = true; + } else { + // Some other error + isBare = false; // At least try to look into submodules... + isBare = false; + } + + LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to determine " + + "isBareRepository() in '" + referenceBaseDirAbs + "'; " + + "assuming '" + isBare + "': " + e.toString()); } // Simplify checks below by stating a useless needle is null From 4ee6935f48f6d1ad98fc5c4063e009238de104d6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 27 Jan 2022 00:06:34 +0100 Subject: [PATCH 116/155] JGitAPIImpl.java + CliGitAPIImpl.java: report "reference" pathname that may take a few git queries to resolve --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +++-- .../java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index dd1ded53d7..3f59bffa6d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -769,8 +769,9 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + - "it may take a few git queries logged below to resolve it into a particular directory name"); + listener.getLogger().println("[INFO] The git reference repository path '" + + reference + "' is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index c407f7b1ec..63707260b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1458,8 +1458,9 @@ public void execute() throws GitException, InterruptedException { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + - "it may take a few git queries logged below to resolve it into a particular directory name"); + listener.getLogger().println("[INFO] The git reference repository path '" + + reference + "'is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { From 558d07600423882c450277a28501c010396be416 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Aug 2022 11:52:28 +0200 Subject: [PATCH 117/155] [JENKINS-69193] GitAPITestCase: test_submodule_update_with_error(): rely less on strict wording of messages from git CLI tool --- .../jenkinsci/plugins/gitclient/GitAPITestCase.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java index 1d0716d789..bee3f8fc80 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java @@ -1225,7 +1225,16 @@ public void test_submodule_update_with_error() throws Exception { w.git.submoduleUpdate().execute(); fail("Did not throw expected submodule update exception"); } catch (GitException e) { - assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" returned status code 1")); + /* Depending on git program implementation/version, the string can be either short: + * Command "git submodule update modules/ntp" returned status code 1" + * or detailed: + * Command "git submodule update modules/ntp" executed in workdir "C:\Users\..." returned status code 1 + * so we catch below the two common parts separately. + * NOTE: git codebase itself goes to great extents to forbid their + * own unit-testing code from relying on emitted text messages. + */ + assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" ")); + assertThat(e.getMessage(), containsString(" returned status code 1")); } } From 4402820b8029493267006e3c34176f66d0e58b68 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 10 Aug 2022 10:40:11 +0200 Subject: [PATCH 118/155] [JENKINS-69193] GitAPITestUpdateCliGit: rely less on strict wording of messages from git CLI tool (after PR #824 mass renaming) --- .../plugins/gitclient/GitAPITestUpdateCliGit.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java index 502257c8b2..335341b202 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java @@ -48,7 +48,16 @@ public void testSubmoduleUpdateWithError() throws Exception { w.git.submoduleUpdate().execute(); fail("Did not throw expected submodule update exception"); } catch (GitException e) { - assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" returned status code 1")); + /* Depending on git program implementation/version, the string can be either short: + * Command "git submodule update modules/ntp" returned status code 1" + * or detailed: + * Command "git submodule update modules/ntp" executed in workdir "C:\Users\..." returned status code 1 + * so we catch below the two common parts separately. + * NOTE: git codebase itself goes to great extents to forbid their + * own unit-testing code from relying on emitted text messages. + */ + assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" ")); + assertThat(e.getMessage(), containsString(" returned status code 1")); } } From c8820502a797ad28934708bbb5db66a6e99e6f34 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Tue, 11 Apr 2023 12:18:54 -0600 Subject: [PATCH 119/155] Format source changes --- src/main/java/hudson/plugins/git/GitAPI.java | 2 +- .../plugins/gitclient/CliGitAPIImpl.java | 56 +- .../plugins/gitclient/GitClient.java | 1 - .../plugins/gitclient/JGitAPIImpl.java | 26 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 512 +++++++++++------- .../plugins/gitclient/GitClientCloneTest.java | 173 +++--- .../plugins/gitclient/WorkspaceWithRepo.java | 3 +- 7 files changed, 462 insertions(+), 311 deletions(-) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index adb36d73e0..506f0a6dbe 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -175,7 +175,7 @@ public GitClient subGit(String subdir) { /** {@inheritDoc} */ public GitClient newGit(String somedir) { - return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); + return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); } /** {@inheritDoc} */ diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 9929eb21e5..12c518334d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -836,13 +836,16 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path '" + - reference + "' is parameterized, it may take a few git queries logged " + - "below to resolve it into a particular directory name"); + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + + "' is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + listener.getLogger() + .println("[ERROR] Could not make File object from reference path, skipping its use: " + + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -865,12 +868,17 @@ else if (!referencePath.isDirectory()) else { File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + listener.getLogger() + .println( + "[WARNING] Reference path does not contain an objects directory (not a git repo?): " + + objectsPath); else { // Go behind git's back to write a meta file in new workspace File alternates = new File(workspace, ".git/objects/info/alternates"); - try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + try (PrintWriter w = new PrintWriter( + alternates, Charset.defaultCharset().toString())) { + String absoluteReference = + objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use w.print(absoluteReference); @@ -1514,8 +1522,7 @@ public void execute() throws GitException, InterruptedException { listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else - args.add("--reference", ref); + else args.add("--reference", ref); } // else handled below in per-module loop } if (shallow) { @@ -1577,7 +1584,10 @@ else if (!referencePath.isDirectory()) if (isParameterizedReferenceRepository(ref)) { File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); + listener.getLogger() + .println( + "[ERROR] Could not make File object from reference path, skipping its use: " + + ref); } else { String expRef = null; if (referencePath.getPath().equals(ref)) { @@ -1590,8 +1600,7 @@ else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); - else - args.add("--reference", referencePath.getPath()); + else args.add("--reference", referencePath.getPath()); } } @@ -1709,7 +1718,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr /** {@inheritDoc} */ @Override public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { - String result = launchCommand( "config", "--local", "--list" ); + String result = launchCommand("config", "--local", "--list"); Map uriNames = new HashMap<>(); for (String line : result.split("\\R+")) { line = StringUtils.trim(line); @@ -1729,7 +1738,8 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); uriNames.put(uSafe.toString(), remoteName); uriNames.put(uSafe.toASCIIString(), remoteName); - } catch (URISyntaxException ue) {} // ignore, move along + } catch (URISyntaxException ue) { + } // ignore, move along } return uriNames; } @@ -1737,7 +1747,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr /** {@inheritDoc} */ @Override public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { - String result = launchCommand( "config", "--local", "--list" ); + String result = launchCommand("config", "--local", "--list"); Map uriNames = new HashMap<>(); for (String line : result.split("\\R+")) { line = StringUtils.trim(line); @@ -1755,7 +1765,8 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); uriNames.put(uSafe.toString(), remoteName); uriNames.put(uSafe.toASCIIString(), remoteName); - } catch (URISyntaxException ue) {} // ignore, move along + } catch (URISyntaxException ue) { + } // ignore, move along } return uriNames; } @@ -2944,11 +2955,14 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e if (status != 0) { if (workDir == null) - workDir = java.nio.file.Paths.get(".").toAbsolutePath().normalize().toFile(); - throw new GitException("Command \"" + command + - "\" executed in workdir \"" + workDir.toString() + - "\" returned status code " + status + - ":\nstdout: " + stdout + "\nstderr: " + stderr); + workDir = java.nio.file.Paths.get(".") + .toAbsolutePath() + .normalize() + .toFile(); + throw new GitException("Command \"" + command + "\" executed in workdir \"" + + workDir.toString() + "\" returned status code " + + status + ":\nstdout: " + + stdout + "\nstderr: " + stderr); } return stdout; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 49342863cb..5111804ff9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -719,7 +719,6 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat */ List revList(String ref) throws GitException, InterruptedException; - // --- new instance of same applied class /** diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 14750bab2c..978f7473b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1555,14 +1555,19 @@ public void execute() throws GitException { if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { - // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path '" + - reference + "'is parameterized, it may take a few git queries logged " + - "below to resolve it into a particular directory name"); + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via + // listener + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + + "'is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + listener.getLogger() + .println( + "[ERROR] Could not make File object from reference path, skipping its use: " + + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -1581,16 +1586,21 @@ public void execute() throws GitException { if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); + listener.getLogger() + .println("[WARNING] Reference path is not a directory: " + reference); else { File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + listener.getLogger() + .println( + "[WARNING] Reference path does not contain an objects directory (no git repo?): " + + objectsPath); else { // Go behind git's back to write a meta file in new workspace try { File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + String absoluteReference = + objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2fb4a100e6..02eeffb3de 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1,9 +1,8 @@ package org.jenkinsci.plugins.gitclient; import static java.util.Arrays.copyOfRange; -import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; -import hudson.FilePath; + import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; @@ -11,32 +10,26 @@ import hudson.plugins.git.Revision; import hudson.plugins.git.Tag; import hudson.remoting.Channel; - -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RefSpec; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.net.URISyntaxException; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; - +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; import org.jenkinsci.plugins.gitclient.verifier.HostKeyVerifierFactory; /** @@ -221,11 +214,9 @@ public static File getObjectsFile(File reference) { return reference; } - if (!reference.exists()) - return null; + if (!reference.exists()) return null; - if (!reference.isDirectory()) - return null; + if (!reference.isDirectory()) return null; File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those File objects = null; @@ -233,13 +224,15 @@ public static File getObjectsFile(File reference) { if (fGit.exists()) { if (fGit.isDirectory()) { objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below -/* - if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null - return objects; // Some Java error, could not make object from the paths involved - } -*/ - LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + - fGit.getAbsolutePath().toString() + "' which is a directory"); + /* + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null + return objects; // Some Java error, could not make object from the paths involved + } + */ + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -248,24 +241,27 @@ public static File getObjectsFile(File reference) { // to the index and other metadata stored in its "parent" // repository's directory: // "gitdir: ../.git/modules/childRepoName" - LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + - fGit.getAbsolutePath().toString() + "' which is NOT a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); - while ((line = reader.readLine()) != null) - { + while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); - if (parts.length >= 2) - { + if (parts.length >= 2) { String key = parts[0].trim(); String value = parts[1].trim(); if (key.equals("gitdir")) { objects = new File(reference, value); - LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + - fGit.getAbsolutePath().toString() + "', found reference to objects " + - "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINE, + "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + + "', found reference to objects " + "which should be at: '" + + objects.getAbsolutePath().toString() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string // processing), but callers might benefit from canonicising and ensuring @@ -276,15 +272,23 @@ public static File getObjectsFile(File reference) { // prefixes for the contexts the callers would define for themselves. break; } - LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + - fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); } } if (objects == null) { - LOGGER.log(Level.WARNING, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + LOGGER.log( + Level.WARNING, + "getObjectsFile(): failed to parse '" + + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); } } catch (IOException e) { - LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getObjectsFile(): failed to parse '" + + fGit.getAbsolutePath().toString() + "': " + e.toString()); objects = null; } try { @@ -292,43 +296,52 @@ public static File getObjectsFile(File reference) { reader.close(); } } catch (IOException e) { - LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getObjectsFile(): failed to close file after parsing '" + + fGit.getAbsolutePath().toString() + "': " + e.toString()); } } } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + - fGit.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): did not find any checked-out '" + + fGit.getAbsolutePath().toString() + "'"); } if (objects == null || !objects.isDirectory()) { // either reference path is bare repo (no ".git" inside), // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare -/* - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved - } -*/ + /* + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } + */ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { if (objects.isDirectory()) { - LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + - objects.getAbsolutePath().toString() + "' which is a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is a directory"); } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + - objects.getAbsolutePath().toString() + "' which is NOT a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is NOT a directory"); } } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + - objects.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): did not find any bare '" + + objects.getAbsolutePath().toString() + "'"); } } - if (!objects.exists()) - return null; + if (!objects.exists()) return null; - if (!objects.isDirectory()) - return null; + if (!objects.isDirectory()) return null; // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath @@ -383,15 +396,13 @@ public static Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ - @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", - justification = "Path operations below intentionally use absolute '/' in some cases" - ) + @SuppressFBWarnings( + value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { - if (!url.startsWith("/") && - !url.startsWith(".") - ) { + if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname if (checkLocalPaths) { File urlPath = new File(url); @@ -399,16 +410,30 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { try { // Check if the string in urlNormalized is a valid // relative path (subdir) in current working directory - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); } catch (java.nio.file.InvalidPathException ipe1) { - // e.g. Illegal char <:> at index 30: C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone + // e.g. Illegal char <:> at index 30: + // C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone try { // Re-check in another manner - urlNormalized = "file://" + Paths.get( Paths.get("", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); } catch (java.nio.file.InvalidPathException ipe2) { // Finally, fall back to checking the originally // fully-qualified path - urlNormalized = "file://" + Paths.get( Paths.get("/", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("/", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); } } } else { @@ -424,12 +449,17 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { if (url.startsWith("/")) { urlNormalized = "file://" + urlNormalized; } else { - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); + ; } } } - LOGGER.log(Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + LOGGER.log( + Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -473,7 +503,8 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * when recursing, since this directory was checked already * as part of parent directory inspection. */ - public SimpleEntry> getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { + public SimpleEntry> getSubmodulesUrls( + String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -486,13 +517,18 @@ public SimpleEntry> getSubmodulesUrls(String re // We want to hit same dirs only once, so canonicize paths below String referenceBaseDirAbs; try { - referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath().toString(); + referenceBaseDirAbs = new File(referenceBaseDir) + .getAbsoluteFile() + .getCanonicalPath() + .toString(); } catch (IOException e) { // Access error while dereferencing some parent?.. referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); - LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to canonicize '" + - referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + e.toString()); - //return new SimpleEntry<>(false, result); + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to canonicize '" + referenceBaseDir + "' => '" + referenceBaseDirAbs + + "': " + e.toString()); + // return new SimpleEntry<>(false, result); } // "this" during a checkout typically represents the job workspace, @@ -502,7 +538,7 @@ public SimpleEntry> getSubmodulesUrls(String re Boolean isBare = false; try { - isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); + isBare = ((hudson.plugins.git.IGitAPI) referenceGit).isBareRepository(); } catch (InterruptedException | GitException e) { // Proposed base directory whose subdirs contain refrepos might // itself be not a repo. Shouldn't be per reference scripts, but... @@ -519,9 +555,11 @@ public SimpleEntry> getSubmodulesUrls(String re isBare = false; } - LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to determine " + - "isBareRepository() in '" + referenceBaseDirAbs + "'; " + - "assuming '" + isBare + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to determine " + "isBareRepository() in '" + + referenceBaseDirAbs + "'; " + "assuming '" + + isBare + "': " + e.toString()); } // Simplify checks below by stating a useless needle is null @@ -585,36 +623,57 @@ public SimpleEntry> getSubmodulesUrls(String re arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (getObjectsFile(f) != null) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking for submodule URL needle='" + needle + + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking git workspace in dir '" + + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); } } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); } } } @@ -643,65 +702,65 @@ public SimpleEntry> getSubmodulesUrls(String re checkedDirs.add(resultEntry[0]); } -/* -// TBD: Needs a way to list submodules in given workspace and convert -// that into (relative) subdirs, possibly buried some levels deep, for -// cases where the submodule is defined in parent with the needle URL. -// Maybe merge with current "if isBare" below, to optionally seed -// same arrDirnames with different values and check remotes listed -// in those repos. - // If current repo *is NOT* bare - check its submodules - // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory. - // But we can add at higher priority submodule path(s) whose - // basename of the URL matches the needleBasename. And then - // other submodule paths to inspect before arbitrary subdirs. - if (!isBare) { - try { - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // * if we are recursed into a "leaf" project and inspect ITS - // submodules, look at all git tips or even commits, to find - // and inspect all unique (by hash) .gitmodule objects, since - // over time or in different branches a "leaf" project could - // reference different subs? - // getRemoteUrls() => Map -// arrDirnames.clear(); - - // TODO: Check subdirs that are git workspaces, and remove "|| true" above -// LinkedHashSet checkedDirs = new LinkedHashSet<>(); -// for (String[] resultEntry : result) { -// checkedDirs.add(resultEntry[0]); -// } - - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; + /* + // TBD: Needs a way to list submodules in given workspace and convert + // that into (relative) subdirs, possibly buried some levels deep, for + // cases where the submodule is defined in parent with the needle URL. + // Maybe merge with current "if isBare" below, to optionally seed + // same arrDirnames with different values and check remotes listed + // in those repos. + // If current repo *is NOT* bare - check its submodules + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. + if (!isBare) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? + // getRemoteUrls() => Map + // arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above + // LinkedHashSet checkedDirs = new LinkedHashSet<>(); + // for (String[] resultEntry : result) { + // checkedDirs.add(resultEntry[0]); + // } + + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } - } catch (Exception e) { - // ignore, go to next slide - } - } -*/ + */ // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, @@ -735,10 +794,12 @@ public SimpleEntry> getSubmodulesUrls(String re } } - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + - ((isBare ? "" : "submodules first, then ")) + - "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + - "' per absolute arrDirnames: " + arrDirnames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at " + ((isBare ? "" : "submodules first, then ")) + + "all subdirs that have a .git, under refrepo '" + + referenceBaseDirAbs + "' per absolute arrDirnames: " + + arrDirnames.toString()); for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes @@ -750,19 +811,34 @@ public SimpleEntry> getSubmodulesUrls(String re // but better be on the safe side :) if (!checkedDirs.contains(dirname)) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking " + + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + + "in existing refrepo dir '" + dirname + "'"); GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking git workspace in dir '" + + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); - if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); + if (needle != null + && needleNorm != null + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } // Cache the finding to return eventually, for each remote: @@ -770,11 +846,15 @@ public SimpleEntry> getSubmodulesUrls(String re // * original remote URI from that workspace's config // * normalized remote URI // * name of the remote from that workspace's config ("origin" etc) - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); } } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); } } @@ -782,14 +862,20 @@ public SimpleEntry> getSubmodulesUrls(String re // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry > subEntriesRet = getSubmodulesUrls(dirname, needle, false); + SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! - LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -811,13 +897,13 @@ public SimpleEntry> getSubmodulesUrls(String re // results looks like it is related to the needle. LinkedHashSet resultFiltered = new LinkedHashSet<>(); -/* - if (!checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once, - // in the parent dir? - return new SimpleEntry<>(false, resultFiltered); - } -*/ + /* + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); + } + */ // Separate lists by suggestion priority: // 1) URI basename similarity @@ -833,9 +919,9 @@ public SimpleEntry> getSubmodulesUrls(String re for (String[] subEntry : result) { // Iterating to filter suggestions in order of original // directory-walk prioritization under current reference - String dirName = subEntry[0]; - String uri = subEntry[1]; - String uriNorm = subEntry[2]; + String dirName = subEntry[0]; + String uri = subEntry[1]; + String uriNorm = subEntry[2]; String remoteName = subEntry[3]; Integer sep; String uriNormBasename; @@ -958,10 +1044,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() && - isParameterizedReferenceRepository(reference) && - url != null && !url.isEmpty() - ) { + if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); @@ -986,31 +1069,36 @@ public File findParameterizedReferenceRepository(String reference, String url) { String urlNormalized = normalizeGitUrl(url, true); // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference='" + reference + "'\n" + - "url='" + url + "'\n" + - "urlNormalized='" + urlNormalized + "'\n"); + System.err.println("reference='" + reference + "'\n" + "url='" + + url + "'\n" + "urlNormalized='" + + urlNormalized + "'\n"); // Let users know why there are many "git config --list" lines in their build log: - LOGGER.log(Level.INFO, "Trying to resolve parameterized Git reference repository '" + - reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); + LOGGER.log( + Level.INFO, + "Trying to resolve parameterized Git reference repository '" + reference + + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); String referenceExpanded = null; if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_SHA256\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: // it allows to gradually convert a big combined reference // repository into smaller chunks without breaking builds. - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } - } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + } else if (reference.endsWith("/${GIT_URL_BASENAME}") + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1024,18 +1112,20 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); - if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + if (url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } } - if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (!url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1046,18 +1136,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + if (getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1078,7 +1167,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass the unmodified "url" value here, the routine // differentiates original spelling vs. normalization while // looking for its needle in the haystack. - SimpleEntry > subEntriesRet = getSubmodulesUrls(referenceBaseDir, url, true); + SimpleEntry> subEntriesRet = + getSubmodulesUrls(referenceBaseDir, url, true); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); if (!subEntries.isEmpty()) { @@ -1098,8 +1188,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (referenceExpanded == null) { referenceExpanded = subEntries.iterator().next()[0]; } - LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); - if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + LOGGER.log( + Level.FINE, + "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + + "' from subEntries"); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1119,13 +1214,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - } - else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1142,8 +1236,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); - LOGGER.log(Level.INFO, "After resolving the parameterized Git reference repository, " + - "decided to use '" + reference + "' directory for URL '" + url + "'"); + LOGGER.log( + Level.INFO, + "After resolving the parameterized Git reference repository, " + "decided to use '" + reference + + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 97fa5c6faa..f6814fade4 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -12,20 +12,6 @@ import hudson.plugins.git.GitException; import hudson.remoting.VirtualChannel; import hudson.util.StreamTaskListener; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.eclipse.jgit.lib.ConfigConstants; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RefSpec; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -38,6 +24,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -280,15 +267,21 @@ public void test_clone_reference() throws Exception { @Test public void test_clone_reference_parameterized_basename() throws Exception, IOException, InterruptedException { - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME}").execute(); + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .reference(workspace.localMirror() + "/${GIT_URL_BASENAME}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: "), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: "), + is(true)); // TODO: Actually construct the local filesystem path to match // the last pathname component from the URL (plus/minus ".git" // extension). Be sure to clean away this path at end of test, @@ -296,31 +289,42 @@ public void test_clone_reference_parameterized_basename() throws Exception, IOEx // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. - //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); - //assertAlternateFilePointsToLocalMirror(); - //assertBranchesExist(testGitClient.getBranches(), "master"); - //assertNoObjectsInRepository(); + // assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference + // repository: "), is(true)); + // assertAlternateFilePointsToLocalMirror(); + // assertBranchesExist(testGitClient.getBranches(), "master"); + // assertNoObjectsInRepository(); } @Test - public void test_clone_reference_parameterized_basename_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_basename_fallback() + throws Exception, IOException, InterruptedException { // TODO: Currently we do not make paths which would invalidate // this test, but note the test above might do just that later. - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}").execute(); + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), + is(true)); // With fallback mode, and nonexistent parameterized reference // repository, and a usable repository in the common path (what // remains if the parameterizing suffix is just discarded), this // common path should be used. So it should overall behave same // as the non-parameterized test_clone_reference_basename() above. - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalMirror(); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); @@ -331,7 +335,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsMirror = workspace.localMirror(); /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java * should be okay for network URLs but are too complex for local pathnames */ - //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); @@ -344,10 +348,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { - /* Note: per parser of magic suffix, use slash - not OS separator char - * And be sure to use relative paths here (see - * WorkspaceWithRepo.java::localMirror()): - */ + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { @@ -357,10 +361,14 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + - "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); - testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); @@ -370,27 +378,33 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages + - (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: ") && - (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")), + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: ") + && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)), + is(true)); if (wsRefrepo != null) { - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path + // structure } @Test - public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_sha256_fallback() + throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java * should be okay for network URLs but are too complex for local pathnames */ - //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); @@ -403,10 +417,10 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { - /* Note: per parser of magic suffix, use slash - not OS separator char - * And be sure to use relative paths here (see - * WorkspaceWithRepo.java::localMirror()): - */ + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { @@ -416,10 +430,14 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + - "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); - testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}").execute(); + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); @@ -431,20 +449,24 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio // Note: we do not expect the closing single quote after wsRefrepoBase // because other tests might pollute our test area, and SHA dir is there - assertThat("Reference repo name-parsing logged in: " + messages + - "\n...and replaced with: '" + wsRefrepoBase, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages + "\n...and replaced with: '" + wsRefrepoBase, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), + is(true)); // Barring filesystem errors, if we have the "custom" refrepo // we expect it to be used (fallback mode is not triggered) if (wsRefrepo != null) { - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path + // structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -640,16 +662,20 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOExceptio assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { - final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) + throws IOException, InterruptedException { + final String alternates = + ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); } @@ -662,18 +688,23 @@ private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) + throws IOException, InterruptedException { final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); } - private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) throws IOException, InterruptedException { - assertThat("Did not find '" + alternates + "' under '" + _testGitDir + "'", - new File(_testGitDir, alternates), is(anExistingFile())); + private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) + throws IOException, InterruptedException { + assertThat( + "Did not find '" + alternates + "' under '" + _testGitDir + "'", + new File(_testGitDir, alternates), + is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); assertThat("Alternates file content", actualContent, is(expectedContent)); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 6d316aa267..1f225036e1 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -142,7 +142,8 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * deleteRecursive() will discard a clone that * 'lost the race'. */ - System.err.println("removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); + System.err.println( + "removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } } else { From 9863085827f04abe753045b0dd8acd16fa0f1703 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Tue, 11 Apr 2023 15:11:50 -0600 Subject: [PATCH 120/155] Fix merge mistake --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 02eeffb3de..8dd94c6385 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -3,6 +3,7 @@ import static java.util.Arrays.copyOfRange; import static org.apache.commons.lang.StringUtils.join; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; From eeb47b42383bca343bb52f16901e020617c36b9b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 14 Jun 2023 23:50:44 +0200 Subject: [PATCH 121/155] GitClient.java: minor fix to javadoc for subGit() --- src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 5111804ff9..54c0ac5644 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -735,7 +735,7 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat /** * subGit. * - * @return a IGitAPI implementation to manage git submodule repository + * @return an {@link IGitAPI} implementation to manage git submodule repository * @param subdir a {@link java.lang.String} object. */ GitClient subGit(String subdir); From 9f162aaa352610ac2837836c6681c31c2a67f8bd Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:24:43 +0200 Subject: [PATCH 122/155] Reconcile code style with changes in the PR #644 (added by @markewaite) and fix some Keeping an eye out for comprehensibility of the resulting source code text --- .../plugins/gitclient/CliGitAPIImpl.java | 59 ++-- .../plugins/gitclient/JGitAPIImpl.java | 32 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 274 ++++++++++-------- .../plugins/gitclient/GitClientCloneTest.java | 39 ++- 4 files changed, 232 insertions(+), 172 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index d50e1568c1..a65efcffc0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -837,15 +837,15 @@ public void execute() throws GitException, InterruptedException { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener listener.getLogger() - .println("[INFO] The git reference repository path '" + reference - + "' is parameterized, it may take a few git queries logged " + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger() - .println("[ERROR] Could not make File object from reference path, skipping its use: " - + reference); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -861,18 +861,17 @@ public void execute() throws GitException, InterruptedException { reference = referencePath.getPath(); } - if (!referencePath.exists()) + if (!referencePath.exists()) { listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) + } else if (!referencePath.isDirectory()) { listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { + } else { File objectsPath = getObjectsFile(referencePath); - if (objectsPath == null || !objectsPath.isDirectory()) + if (objectsPath == null || !objectsPath.isDirectory()) { listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (not a git repo?): " - + objectsPath); - else { + .println("[WARNING] Reference path does not contain an objects directory " + + "(not a git repo?): " + objectsPath); + } else { // Go behind git's back to write a meta file in new workspace File alternates = new File(workspace, ".git/objects/info/alternates"); try (PrintWriter w = new PrintWriter( @@ -1518,11 +1517,13 @@ public void execute() throws GitException, InterruptedException { if ((ref != null) && !ref.isEmpty()) { if (!isParameterizedReferenceRepository(ref)) { File referencePath = new File(ref); - if (!referencePath.exists()) + if (!referencePath.exists()) { listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); - else if (!referencePath.isDirectory()) + } else if (!referencePath.isDirectory()) { listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else args.add("--reference", ref); + } else { + args.add("--reference", ref); + } } // else handled below in per-module loop } if (shallow) { @@ -1585,9 +1586,8 @@ else if (!referencePath.isDirectory()) File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { listener.getLogger() - .println( - "[ERROR] Could not make File object from reference path, skipping its use: " - + ref); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + ref); } else { String expRef = null; if (referencePath.getPath().equals(ref)) { @@ -1596,11 +1596,15 @@ else if (!referencePath.isDirectory()) expRef = referencePath.getPath(); expRef += " (expanded from " + ref + ")"; } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); - else args.add("--reference", referencePath.getPath()); + if (!referencePath.exists()) { + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + expRef); + } else if (!referencePath.isDirectory()) { + listener.getLogger() + .println("[WARNING] Reference path is not a directory: " + expRef); + } else { + args.add("--reference", referencePath.getPath()); + } } } @@ -2959,10 +2963,11 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e .toAbsolutePath() .normalize() .toFile(); - throw new GitException("Command \"" + command + "\" executed in workdir \"" - + workDir.toString() + "\" returned status code " - + status + ":\nstdout: " - + stdout + "\nstderr: " + stderr); + throw new GitException("Command \"" + command + + "\" executed in workdir \"" + workDir.toString() + + "\" returned status code " + status + + ":\nstdout: " + stdout + + "\nstderr: " + stderr); } return stdout; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 978f7473b2..44c95dac05 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1555,19 +1555,18 @@ public void execute() throws GitException { if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { - // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via - // listener + // LegacyCompatibleGitAPIImpl.java has a logging trace, + // but not into build console via listener listener.getLogger() - .println("[INFO] The git reference repository path '" + reference - + "'is parameterized, it may take a few git queries logged " + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger() - .println( - "[ERROR] Could not make File object from reference path, skipping its use: " - + reference); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -1583,26 +1582,25 @@ public void execute() throws GitException { reference = referencePath.getPath(); } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) + if (!referencePath.exists()) { + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + reference); + } else if (!referencePath.isDirectory()) { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); - else { + } else { File objectsPath = getObjectsFile(referencePath); - if (objectsPath == null || !objectsPath.isDirectory()) + if (objectsPath == null || !objectsPath.isDirectory()) { listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (no git repo?): " - + objectsPath); - else { + .println("[WARNING] Reference path does not contain an objects directory " + + "(no git repo?): " + objectsPath); + } else { // Go behind git's back to write a meta file in new workspace try { File alternates = new File(workspace, ".git/objects/info/alternates"); String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { // git implementations on windows also use w.print(absoluteReference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8dd94c6385..9e60a3dc6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -215,9 +215,13 @@ public static File getObjectsFile(File reference) { return reference; } - if (!reference.exists()) return null; + if (!reference.exists()) { + return null; + } - if (!reference.isDirectory()) return null; + if (!reference.isDirectory()) { + return null; + } File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those File objects = null; @@ -226,9 +230,9 @@ public static File getObjectsFile(File reference) { if (fGit.isDirectory()) { objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below /* - if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null - return objects; // Some Java error, could not make object from the paths involved - } + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null + return objects; // Some Java error, could not make object from the paths involved + } */ LOGGER.log( Level.FINEST, @@ -261,7 +265,7 @@ public static File getObjectsFile(File reference) { Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath().toString() - + "', found reference to objects " + "which should be at: '" + + "', found reference to objects which should be at: '" + objects.getAbsolutePath().toString() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string @@ -315,9 +319,9 @@ public static File getObjectsFile(File reference) { // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare /* - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved - } + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } */ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { @@ -340,9 +344,13 @@ public static File getObjectsFile(File reference) { } } - if (!objects.exists()) return null; + if (!objects.exists()) { + return null; + } - if (!objects.isDirectory()) return null; + if (!objects.isDirectory()) { + return null; + } // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath @@ -454,13 +462,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) .normalize() .toString(); - ; } } } LOGGER.log( - Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + Level.FINEST, + "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -527,8 +535,9 @@ public SimpleEntry> getSubmodulesUrls( referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); LOGGER.log( Level.SEVERE, - "getSubmodulesUrls(): failed to canonicize '" + referenceBaseDir + "' => '" + referenceBaseDirAbs - + "': " + e.toString()); + "getSubmodulesUrls(): failed to canonicize '" + + referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + + e.toString()); // return new SimpleEntry<>(false, result); } @@ -558,9 +567,9 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.SEVERE, - "getSubmodulesUrls(): failed to determine " + "isBareRepository() in '" - + referenceBaseDirAbs + "'; " + "assuming '" - + isBare + "': " + e.toString()); + "getSubmodulesUrls(): failed to determine isBareRepository() in '" + + referenceBaseDirAbs + "'; " + "assuming '" + isBare + "': " + + e.toString()); } // Simplify checks below by stating a useless needle is null @@ -626,8 +635,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs - + "', per arrDirnames: " + arrDirnames.toString()); + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -656,8 +665,8 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm - + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); @@ -710,57 +719,56 @@ public SimpleEntry> getSubmodulesUrls( // Maybe merge with current "if isBare" below, to optionally seed // same arrDirnames with different values and check remotes listed // in those repos. - // If current repo *is NOT* bare - check its submodules - // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory. - // But we can add at higher priority submodule path(s) whose - // basename of the URL matches the needleBasename. And then - // other submodule paths to inspect before arbitrary subdirs. - if (!isBare) { - try { - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // * if we are recursed into a "leaf" project and inspect ITS - // submodules, look at all git tips or even commits, to find - // and inspect all unique (by hash) .gitmodule objects, since - // over time or in different branches a "leaf" project could - // reference different subs? - // getRemoteUrls() => Map - // arrDirnames.clear(); - - // TODO: Check subdirs that are git workspaces, and remove "|| true" above - // LinkedHashSet checkedDirs = new LinkedHashSet<>(); - // for (String[] resultEntry : result) { - // checkedDirs.add(resultEntry[0]); - // } - - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - } - } catch (Exception e) { - // ignore, go to next slide + // If current repo *is NOT* bare - check its submodules + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. + if (!isBare) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? + // getRemoteUrls() => Map + //// arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above + //// LinkedHashSet checkedDirs = new LinkedHashSet<>(); + //// for (String[] resultEntry : result) { + //// checkedDirs.add(resultEntry[0]); + //// } + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } + } catch (Exception e) { + // ignore, go to next slide + } + } */ // If current repo *is* bare (can't have proper submodules), or if the @@ -797,7 +805,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at " + ((isBare ? "" : "submodules first, then ")) + "getSubmodulesUrls(): looking at " + + ((isBare ? "" : "submodules first, then ")) + "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); @@ -805,7 +814,9 @@ public SimpleEntry> getSubmodulesUrls( for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above // by getObjectsFile(). Probably should not check exists/dir either, @@ -833,11 +844,12 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm - + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + + "' (uriNorm='" + uriNorm + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri))) { + && (needleNorm.equals(uriNorm) || needle.equals(uri)) + ) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); @@ -862,21 +874,24 @@ public SimpleEntry> getSubmodulesUrls( // Here is a good spot to recurse this routine into a // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. - LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + SimpleEntry> subEntriesRet = + getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " - + subEntries.size() + " found mappings"); + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + + "' with " + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! LOGGER.log( Level.FINE, - "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname - + "': " + subEntries.iterator().next()[0]); + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + + dirname + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -899,11 +914,11 @@ public SimpleEntry> getSubmodulesUrls( LinkedHashSet resultFiltered = new LinkedHashSet<>(); /* - if (!checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once, - // in the parent dir? - return new SimpleEntry<>(false, resultFiltered); - } + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); + } */ // Separate lists by suggestion priority: @@ -1045,17 +1060,25 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { + if (!referencePath.exists() + && isParameterizedReferenceRepository(reference) + && url != null + && !url.isEmpty() + ) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); File referenceBaseDirFile = new File(referenceBaseDir); if (!referenceBaseDirFile.exists()) { - LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); + LOGGER.log( + Level.WARNING, + "Base Git reference directory " + referenceBaseDir + " does not exist"); return null; } if (!referenceBaseDirFile.isDirectory()) { - LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); + LOGGER.log( + Level.WARNING, + "Base Git reference directory " + referenceBaseDir + " is not a directory"); return null; } @@ -1070,9 +1093,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { String urlNormalized = normalizeGitUrl(url, true); // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference='" + reference + "'\n" + "url='" - + url + "'\n" + "urlNormalized='" - + urlNormalized + "'\n"); + System.err.println("reference='" + reference + "'\n" + + "url='" + url + "'\n" + + "urlNormalized='" + urlNormalized + "'\n"); // Let users know why there are many "git config --list" lines in their build log: LOGGER.log( @@ -1094,12 +1117,15 @@ public File findParameterizedReferenceRepository(String reference, String url) { referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") - || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") + ) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1113,12 +1139,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } @@ -1126,7 +1157,8 @@ && getObjectsFile(referenceExpanded + ".git") == null) { if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1137,17 +1169,24 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") + || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + ) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1191,16 +1230,19 @@ && getObjectsFile(referenceExpanded + ".git") == null) { } LOGGER.log( Level.FINE, - "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded - + "' from subEntries"); + "findParameterizedReferenceRepository(): got referenceExpanded='" + + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else { - LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); + LOGGER.log( + Level.FINE, + "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as for // the refrepo forks' co-hosting friendly basename search), @@ -1215,12 +1257,18 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); - } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", + needleBasename); + } + else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", + needleBasename); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1239,8 +1287,8 @@ && getObjectsFile(referenceExpanded + ".git") == null) { LOGGER.log( Level.INFO, - "After resolving the parameterized Git reference repository, " + "decided to use '" + reference - + "' directory for URL '" + url + "'"); + "After resolving the parameterized Git reference repository, " + + "decided to use '" + reference + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index f6814fade4..fa82bf2c99 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -289,11 +289,14 @@ public void test_clone_reference_parameterized_basename() throws Exception, IOEx // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. - // assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference - // repository: "), is(true)); - // assertAlternateFilePointsToLocalMirror(); - // assertBranchesExist(testGitClient.getBranches(), "master"); - // assertNoObjectsInRepository(); + /* + assertThat("Reference repo logged in: " + messages, + handler.containsMessageSubstring( + "Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + */ } @Test @@ -361,7 +364,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -394,8 +398,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path - // structure + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure } @Test @@ -430,7 +434,8 @@ public void test_clone_reference_parameterized_sha256_fallback() // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -465,8 +470,8 @@ public void test_clone_reference_parameterized_sha256_fallback() assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path - // structure + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -654,11 +659,13 @@ private void assertNoObjectsInRepository() { } // Most tests use this method, expecting a non-bare repo - private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } @@ -680,11 +687,13 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, } // Similar for bare repos, without ".git/" dir - private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(testGitDir); } - private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } From d3a6d569c41da8f51b2d1b61e0c4e8edf1eb3683 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:25:52 +0200 Subject: [PATCH 123/155] Mark @Override methods like in master branch --- src/main/java/hudson/plugins/git/GitAPI.java | 2 ++ .../java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index 506f0a6dbe..e3ef43d0eb 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -174,11 +174,13 @@ public GitClient subGit(String subdir) { } /** {@inheritDoc} */ + @Override public GitClient newGit(String somedir) { return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); } /** {@inheritDoc} */ + @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { if (Git.USE_CLI) { super.setRemoteUrl(name, url); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index b78f6d6e53..962342ca74 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -343,16 +343,19 @@ public String getRemoteUrl(String name) throws GitException, InterruptedExceptio } /** {@inheritDoc} */ + @Override public Map getRemoteUrls() throws GitException, InterruptedException { return proxy.getRemoteUrls(); } /** {@inheritDoc} */ + @Override public Map getRemotePushUrls() throws GitException, InterruptedException { return proxy.getRemotePushUrls(); } /** {@inheritDoc} */ + @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { proxy.setRemoteUrl(name, url); } From 6840c75b75a14f999cd5ef37652a4372768fe6a8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:26:19 +0200 Subject: [PATCH 124/155] Change PrintWriter use of UTF8 like in master branch --- src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 44c95dac05..885d8a693c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1601,7 +1601,7 @@ public void execute() throws GitException { String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); - try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { // git implementations on windows also use w.print(absoluteReference); } From d8ddcdb22e8b17f4f05a3de29d0a5ede74b97321 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:30:01 +0200 Subject: [PATCH 125/155] GitClient.java: minor fix to javadoc for newGit() --- src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 54c0ac5644..8ad8795f4a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -724,7 +724,7 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat /** * newGit. * - * @return a IGitAPI implementation to manage another git repository + * @return an {@link IGitAPI} implementation to manage another git repository * with same general settings and implementation as the current one. * @param somedir a {@link java.lang.String} object. */ From 896c2c6169e9059be273a025e6a2a7362a35fcf9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:30:37 +0200 Subject: [PATCH 126/155] LegacyCompatibleGitAPIImpl: import DigestUtils to use by short reference --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 9e60a3dc6b..55ff32c1b9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.codec.digest.DigestUtils; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -629,7 +630,7 @@ public SimpleEntry> getSubmodulesUrls( arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename + ".git"); } - needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + needleSha = DigestUtils.sha256Hex(needleNorm); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); @@ -1108,7 +1109,7 @@ && isParameterizedReferenceRepository(reference) // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: @@ -1116,7 +1117,7 @@ && isParameterizedReferenceRepository(reference) // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null ) { From 62355d5e13ff236051b7b6a99a83497456af38ec Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:31:18 +0200 Subject: [PATCH 127/155] GitClientCloneTest: drop use of FileUtils.readFileToString() like in master branch --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index fa82bf2c99..61deef1667 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -24,7 +24,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -715,7 +714,7 @@ private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _ new File(_testGitDir, alternates), is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; - final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); + final String actualContent = Files.readString(testGitDir.toPath().resolve(alternates), StandardCharsets.UTF_8); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); From 2c10b594e2fdc2a834fbb61b3bbe2af5cc5eac1e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 01:19:58 +0200 Subject: [PATCH 128/155] spotless is clueless Promulgates inconsistent coding style and breaks "logical" markup of messages defined on multiple lines (like "item " + javavar) which it fails to keep together, etc. Unavoidable evil I guess, but hopefully someone can configure it. --- .../plugins/gitclient/CliGitAPIImpl.java | 6 +- .../plugins/gitclient/JGitAPIImpl.java | 3 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 126 ++++++------------ .../plugins/gitclient/GitClientCloneTest.java | 22 ++- 4 files changed, 53 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index a65efcffc0..298ea0aeed 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1597,11 +1597,9 @@ public void execute() throws GitException, InterruptedException { expRef += " (expanded from " + ref + ")"; } if (!referencePath.exists()) { - listener.getLogger() - .println("[WARNING] Reference path does not exist: " + expRef); + listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); } else if (!referencePath.isDirectory()) { - listener.getLogger() - .println("[WARNING] Reference path is not a directory: " + expRef); + listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); } else { args.add("--reference", referencePath.getPath()); } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 885d8a693c..17fa0c9c7e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1583,8 +1583,7 @@ public void execute() throws GitException { } if (!referencePath.exists()) { - listener.getLogger() - .println("[WARNING] Reference path does not exist: " + reference); + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); } else if (!referencePath.isDirectory()) { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 55ff32c1b9..f85a8cc1f3 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -468,8 +468,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { } LOGGER.log( - Level.FINEST, - "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -636,8 +635,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" - + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -666,8 +665,8 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri - + "' (uriNorm='" + uriNorm + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); @@ -815,9 +814,7 @@ public SimpleEntry> getSubmodulesUrls( for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log( - Level.FINEST, - "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above // by getObjectsFile(). Probably should not check exists/dir either, @@ -845,12 +842,11 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri - + "' (uriNorm='" + uriNorm + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri)) - ) { + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); @@ -875,24 +871,21 @@ public SimpleEntry> getSubmodulesUrls( // Here is a good spot to recurse this routine into a // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. - LOGGER.log( - Level.FINE, - "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry> subEntriesRet = - getSubmodulesUrls(dirname, needle, false); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): returned from recursing into dir '" + dirname - + "' with " + subEntries.size() + " found mappings"); + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! LOGGER.log( Level.FINE, - "getSubmodulesUrls(): got an exact needle match from recursing into dir '" - + dirname + "': " + subEntries.iterator().next()[0]); + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -1061,25 +1054,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() - && isParameterizedReferenceRepository(reference) - && url != null - && !url.isEmpty() - ) { + if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); File referenceBaseDirFile = new File(referenceBaseDir); if (!referenceBaseDirFile.exists()) { - LOGGER.log( - Level.WARNING, - "Base Git reference directory " + referenceBaseDir + " does not exist"); + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); return null; } if (!referenceBaseDirFile.isDirectory()) { - LOGGER.log( - Level.WARNING, - "Base Git reference directory " + referenceBaseDir + " is not a directory"); + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); return null; } @@ -1107,26 +1092,21 @@ && isParameterizedReferenceRepository(reference) String referenceExpanded = null; if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_SHA256\\}$", - DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: // it allows to gradually convert a big combined reference // repository into smaller chunks without breaking builds. - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", DigestUtils.sha256Hex(urlNormalized)); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") - || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") - ) { + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1140,17 +1120,12 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } @@ -1158,8 +1133,7 @@ && getObjectsFile(referenceExpanded + ".git") == null if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1170,24 +1144,17 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") - || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") - ) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1231,19 +1198,16 @@ && getObjectsFile(referenceExpanded + ".git") == null } LOGGER.log( Level.FINE, - "findParameterizedReferenceRepository(): got referenceExpanded='" - + referenceExpanded + "' from subEntries"); + "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else { - LOGGER.log( - Level.FINE, - "findParameterizedReferenceRepository(): got no subEntries"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as for // the refrepo forks' co-hosting friendly basename search), @@ -1258,18 +1222,12 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - } - else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_SUBMODULES\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1288,8 +1246,8 @@ && getObjectsFile(referenceExpanded + ".git") == null LOGGER.log( Level.INFO, - "After resolving the parameterized Git reference repository, " - + "decided to use '" + reference + "' directory for URL '" + url + "'"); + "After resolving the parameterized Git reference repository, " + "decided to use '" + reference + + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 61deef1667..b33c475369 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -363,8 +363,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" - + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -398,7 +397,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, - // with bogus path above and not pre-created path structure + // with bogus path above and not pre-created path structure } @Test @@ -433,8 +432,7 @@ public void test_clone_reference_parameterized_sha256_fallback() // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" - + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -470,7 +468,7 @@ public void test_clone_reference_parameterized_sha256_fallback() assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, - // with bogus path above and not pre-created path structure + // with bogus path above and not pre-created path structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -658,13 +656,11 @@ private void assertNoObjectsInRepository() { } // Most tests use this method, expecting a non-bare repo - private void assertAlternateFilePointsToLocalMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } @@ -686,13 +682,11 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, } // Similar for bare repos, without ".git/" dir - private void assertAlternateFilePointsToLocalBareMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(testGitDir); } - private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } From 4e1f3abb56c42f6b161f9002ebc8c493fa190c2b Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:10:50 -0600 Subject: [PATCH 129/155] Remove redundant null check --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index f1871b4f17..3aba010838 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1577,10 +1577,7 @@ public void execute() throws GitException, InterruptedException { listener.error("Invalid repository for " + sModuleName); throw new GitException("Invalid repository for " + sModuleName); } - String strURIish = null; - if (urIish != null) { - strURIish = urIish.toPrivateString(); - } + String strURIish = urIish.toPrivateString(); if (isParameterizedReferenceRepository(ref)) { File referencePath = findParameterizedReferenceRepository(ref, strURIish); From f844f2397fccf654e689863bcf1b52d043d98e9d Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:31:49 -0600 Subject: [PATCH 130/155] Resolve spotbugs warnings for toString() and unread vars --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 49 +++++++------------ 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f85a8cc1f3..0e9da74446 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -237,8 +237,7 @@ public static File getObjectsFile(File reference) { */ LOGGER.log( Level.FINEST, - "getObjectsFile(): found an fGit '" - + fGit.getAbsolutePath().toString() + "' which is a directory"); + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -249,8 +248,7 @@ public static File getObjectsFile(File reference) { // "gitdir: ../.git/modules/childRepoName" LOGGER.log( Level.FINEST, - "getObjectsFile(): found an fGit '" - + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; @@ -265,9 +263,9 @@ public static File getObjectsFile(File reference) { LOGGER.log( Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" - + fGit.getAbsolutePath().toString() + + fGit.getAbsolutePath() + "', found reference to objects which should be at: '" - + objects.getAbsolutePath().toString() + "'"); + + objects.getAbsolutePath() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string // processing), but callers might benefit from canonicising and ensuring @@ -280,21 +278,20 @@ public static File getObjectsFile(File reference) { } LOGGER.log( Level.FINEST, - "getObjectsFile(): while looking for 'gitdir:' in '" - + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath() + + "', ignoring line: " + line); } } if (objects == null) { LOGGER.log( Level.WARNING, - "getObjectsFile(): failed to parse '" - + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + + "': did not contain a 'gitdir:' entry"); } } catch (IOException e) { LOGGER.log( Level.SEVERE, - "getObjectsFile(): failed to parse '" - + fGit.getAbsolutePath().toString() + "': " + e.toString()); + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + "': " + e.toString()); objects = null; } try { @@ -304,15 +301,12 @@ public static File getObjectsFile(File reference) { } catch (IOException e) { LOGGER.log( Level.SEVERE, - "getObjectsFile(): failed to close file after parsing '" - + fGit.getAbsolutePath().toString() + "': " + e.toString()); + "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath() + "': " + + e.toString()); } } } else { - LOGGER.log( - Level.FINEST, - "getObjectsFile(): did not find any checked-out '" - + fGit.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + fGit.getAbsolutePath() + "'"); } if (objects == null || !objects.isDirectory()) { @@ -329,19 +323,15 @@ public static File getObjectsFile(File reference) { if (objects.isDirectory()) { LOGGER.log( Level.FINEST, - "getObjectsFile(): found a bare '" - + objects.getAbsolutePath().toString() + "' which is a directory"); + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + "' which is a directory"); } else { LOGGER.log( Level.FINEST, - "getObjectsFile(): found a bare '" - + objects.getAbsolutePath().toString() + "' which is NOT a directory"); + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + + "' which is NOT a directory"); } } else { - LOGGER.log( - Level.FINEST, - "getObjectsFile(): did not find any bare '" - + objects.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + objects.getAbsolutePath() + "'"); } } @@ -526,10 +516,7 @@ public SimpleEntry> getSubmodulesUrls( // We want to hit same dirs only once, so canonicize paths below String referenceBaseDirAbs; try { - referenceBaseDirAbs = new File(referenceBaseDir) - .getAbsoluteFile() - .getCanonicalPath() - .toString(); + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath(); } catch (IOException e) { // Access error while dereferencing some parent?.. referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); @@ -930,9 +917,7 @@ public SimpleEntry> getSubmodulesUrls( // Iterating to filter suggestions in order of original // directory-walk prioritization under current reference String dirName = subEntry[0]; - String uri = subEntry[1]; String uriNorm = subEntry[2]; - String remoteName = subEntry[3]; Integer sep; String uriNormBasename; String dirBasename; From 43c38bafcfb6b7505b856f6c91b11e096dd50ad0 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:33:01 -0600 Subject: [PATCH 131/155] Resolve spotbugs warnings on toLowerCase Use English locale as a UTF-8 capable locale --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0e9da74446..19027814bc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -400,7 +401,7 @@ public static Boolean isParameterizedReferenceRepository(String reference) { value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { - String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(Locale.ENGLISH); if (!url.contains("://")) { if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname @@ -605,7 +606,7 @@ public SimpleEntry> getSubmodulesUrls( // FS object. arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); - needleBasenameLC = needleBasename.toLowerCase(); + needleBasenameLC = needleBasename.toLowerCase(Locale.ENGLISH); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC); From 65b55c5d5b50f2cedaf9fc92c651a77e44debe18 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:49:51 -0600 Subject: [PATCH 132/155] Assure stream is closed on exception --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 19027814bc..5f2c4617fe 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -250,10 +250,9 @@ public static File getObjectsFile(File reference) { LOGGER.log( Level.FINEST, "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is NOT a directory"); - BufferedReader reader = null; - try { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8"))) { String line; - reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); if (parts.length >= 2) { @@ -295,16 +294,6 @@ public static File getObjectsFile(File reference) { "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + "': " + e.toString()); objects = null; } - try { - if (reader != null) { - reader.close(); - } - } catch (IOException e) { - LOGGER.log( - Level.SEVERE, - "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath() + "': " - + e.toString()); - } } } else { LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + fGit.getAbsolutePath() + "'"); From 2ff86b7b9776b8fb32df0c6134dcd7a2642b54a9 Mon Sep 17 00:00:00 2001 From: Evgeny Klimov Date: Fri, 11 Apr 2025 11:50:39 +0200 Subject: [PATCH 133/155] src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java: apply mvn:spotless complaint fixes Signed-off-by: Evgeny Klimov --- .../java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 0244fb847d..e145beb3a6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1716,7 +1716,8 @@ public void execute() throws GitException { reference = referencePath.getPath(); } if (!referencePath.exists()) { - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + reference); } else if (!referencePath.isDirectory()) { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); @@ -1731,8 +1732,9 @@ public void execute() throws GitException { // Go behind git's back to write a meta file in new workspace try { File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = - objectsPath.getAbsolutePath().replace('\\', '/'); + String absoluteReference = objectsPath + .getAbsolutePath() + .replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { From 0f8e120a7432d3287c34288673467e07707029be Mon Sep 17 00:00:00 2001 From: Evgeny Klimov Date: Fri, 11 Apr 2025 11:50:55 +0200 Subject: [PATCH 134/155] src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java: port comment from upstream changes Signed-off-by: Evgeny Klimov --- src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index e145beb3a6..802a3ae22b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1722,6 +1722,7 @@ public void execute() throws GitException { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); } else { + // reference path can either be a normal or a base repository File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) { listener.getLogger() From c439d4692d4e68845bcbc66b3e4d5a8582cd4ad8 Mon Sep 17 00:00:00 2001 From: Evgeny Klimov Date: Fri, 11 Apr 2025 11:51:09 +0200 Subject: [PATCH 135/155] src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java: drop duplicate comment from upstream changes Signed-off-by: Evgeny Klimov --- src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 802a3ae22b..bf8638e414 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1737,7 +1737,6 @@ public void execute() throws GitException { .getAbsolutePath() .replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { // git implementations on windows also use w.print(absoluteReference); From 37196d9c19744435551ec464faf80d6504c43ba0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 17 Sep 2025 12:57:40 +0200 Subject: [PATCH 136/155] GitClientCloneTest: fix style complaints with `mvn spotless:apply` Signed-off-by: Jim Klimov --- .../plugins/gitclient/GitClientCloneTest.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b8a0dad828..b0d249c66c 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -302,8 +302,7 @@ void test_clone_reference_parameterized_basename() throws Exception { } @Test - void test_clone_reference_parameterized_basename_fallback() - throws Exception { + void test_clone_reference_parameterized_basename_fallback() throws Exception { // TODO: Currently we do not make paths which would invalidate // this test, but note the test above might do just that later. testGitClient @@ -404,8 +403,7 @@ void test_clone_reference_parameterized_sha256() throws Exception { } @Test - void test_clone_reference_parameterized_sha256_fallback() - throws Exception { + void test_clone_reference_parameterized_sha256_fallback() throws Exception { String wsMirror = workspace.localMirror(); /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java * should be okay for network URLs but are too complex for local pathnames */ @@ -667,13 +665,11 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror() throws Exception assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) - throws Exception { + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws Exception { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) - throws Exception { + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws Exception { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); } @@ -693,13 +689,11 @@ private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) - throws Exception { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws Exception { assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) - throws Exception { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws Exception { final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); } From d3d9a3c3b55a51ef6a763a55974d0227373becd9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 21:58:03 +0200 Subject: [PATCH 137/155] Fix getObjectsFile() returning gitdir directory instead of its objects/ subdir When a workspace's .git is a file (the submodule redirect pattern, e.g. "gitdir: ../../.git/modules/sub"), the code resolved the gitdir path but then used that directory directly as the objects location. Because the gitdir itself is a directory, every downstream isDirectory() check passed, and the alternates file was written with a path like /workspace/.git/modules/sub instead of the correct /workspace/.git/modules/sub/objects Git silently ignores alternates entries that are not an objects/ directory, so the reference repository was never used and a full network fetch happened instead. Fix: append "/objects" to the resolved gitdir path, mirroring what the non-redirect branch already does with new File(fGit, "objects"). Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 9a4284078d..b7462eda90 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -259,7 +259,10 @@ public static File getObjectsFile(File reference) { String key = parts[0].trim(); String value = parts[1].trim(); if (key.equals("gitdir")) { - objects = new File(reference, value); + // value is the gitdir path (e.g. "../../.git/modules/sub"); + // we need the "objects" subdirectory *inside* that gitdir, + // not the gitdir directory itself. + objects = new File(new File(reference, value), "objects"); LOGGER.log( Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" From b24d074c87414eabe3816f20f6411533ef805616 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 21:58:46 +0200 Subject: [PATCH 138/155] Fix GIT_SUBMODULES_FALLBACK branch using wrong regex and never expanding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In findParameterizedReferenceRepository(), the else-branch that handles /${GIT_SUBMODULES_FALLBACK} called: reference.replaceAll("\$\{GIT_SUBMODULES\}$", needleBasename) The pattern targets the non-fallback token, which is not present in the string, so replaceAll() returned the original string unchanged. referenceExpanded therefore still ended with "/${GIT_SUBMODULES_FALLBACK}", getObjectsFile() returned null on that literal path, and the code always fell back to referenceBaseDir — the specific per-URL subdirectory was never tried, defeating the whole point of the fallback mode. Fix: use the correct pattern \$\{GIT_SUBMODULES_FALLBACK\}$ so the token is actually replaced with the computed needleBasename. Co-Authored-By: Claude Sonnet 4.6 --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b7462eda90..da7c893989 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1202,7 +1202,7 @@ && getObjectsFile(referenceExpanded + ".git") == null) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES_FALLBACK\\}$", needleBasename); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { From 1df7ff5a329193970f62d8119271541b957f4cc9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 21:59:40 +0200 Subject: [PATCH 139/155] Remove System.err.println that may expose credentials in build logs findParameterizedReferenceRepository() printed the raw `url` and `urlNormalized` values unconditionally to stderr via System.err.println. Remote URLs can contain embedded credentials (user:token@host), and stderr output bypasses Jenkins' credential-masking facility, potentially leaking secrets into build logs or console output. The same information is already logged at INFO level through the Jenkins task listener earlier in the method, so the debug prints carry no unique value. The stale comment claiming "unit-tests expect this markup on stderr" is incorrect: all test assertions check handler.getMessages() (the Jenkins listener log), not stderr. Remove both System.err.println calls. Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index da7c893989..40fe445ac8 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1056,11 +1056,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); - // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference='" + reference + "'\n" - + "url='" + url + "'\n" - + "urlNormalized='" + urlNormalized + "'\n"); - // Let users know why there are many "git config --list" lines in their build log: LOGGER.log( Level.INFO, @@ -1219,9 +1214,6 @@ && getObjectsFile(referenceExpanded + ".git") == null) { referencePath = new File(reference); } - // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference after='" + reference + "'\n"); - LOGGER.log( Level.INFO, "After resolving the parameterized Git reference repository, " + "decided to use '" + reference From 33b1199353e1fd15fd592f8bed502131f0fd30e4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:00:39 +0200 Subject: [PATCH 140/155] Fix unescaped regex metachar in .git suffix stripping; use newGit() for absolute paths Two related correctness fixes in LegacyCompatibleGitAPIImpl: 1. Regex: every call to replaceAll(".git$", "") used an unescaped dot, which is a regex wildcard matching any character. A URL or directory name ending in e.g. "xgit" or "1git" would have its last four characters incorrectly stripped, producing a wrong basename and causing all SHA256/BASENAME probes to miss even when a correctly-named reference directory exists. Fix: escape to "\.git$" (and "\.[Gg][Ii][Tt]$" for the case-insensitive variant). Same fix applied to the trailing-slash pattern "/*$" -> "/+$" (the original was technically correct but /+ is the right idiom). 2. Absolute-path safety: in the basename-priority probe loop of getSubmodulesUrls(), the code called referenceGit.subGit(dirname) where dirname is an absolute path. subGit() constructs new File(workspace, subdir), which on Windows drops the drive letter when subdir begins with a slash, producing a wrong path on a different drive than the Jenkins workspace. The second loop in the same method already correctly uses this.newGit(dirname). Fix: use this.newGit(dirname) in the first loop too. Co-Authored-By: Claude Sonnet 4.6 --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 40fe445ac8..13f2b3b059 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -393,7 +393,7 @@ public static Boolean isParameterizedReferenceRepository(String reference) { value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { - String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(Locale.ENGLISH); + String urlNormalized = url.replaceAll("/+$", "").replaceAll("\\.git$", "").toLowerCase(Locale.ENGLISH); if (!url.contains("://")) { if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname @@ -578,7 +578,7 @@ public SimpleEntry> getSubmodulesUrls( } else { needleBasename = needle.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".[Gg][Ii][Tt]$", ""); + needleBasename = needleBasename.replaceAll("\\.[Gg][Ii][Tt]$", ""); needleNorm = normalizeGitUrl(needle, true); sep = needleNorm.lastIndexOf("/"); @@ -587,7 +587,7 @@ public SimpleEntry> getSubmodulesUrls( } else { needleNormBasename = needleNorm.substring(sep + 1); } - needleNormBasename = needleNormBasename.replaceAll(".git$", ""); + needleNormBasename = needleNormBasename.replaceAll("\\.git$", ""); // Try with the basename without .git extension, and then with one. // First we try the caller-provided string casing, then normalized. @@ -628,8 +628,12 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle - + "' in existing refrepo subdir '" + dirname + "'"); - GitClient g = referenceGit.subGit(dirname); + + "' in existing abs refrepo subdir '" + dirname + "'"); + // Use newGit() rather than referenceGit.subGit(dirname) + // since dirname is already an absolute path anyway, and + // it seems that going through subGit() may lose the disk + // letter on Windows agents. + GitClient g = this.newGit(dirname); LOGGER.log( Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" @@ -766,7 +770,7 @@ public SimpleEntry> getSubmodulesUrls( // does not denote a directory, or if an I/O error occurs" for (File dir : directories) { if (getObjectsFile(dir) != null) { - String dirname = dir.getPath().replaceAll("/*$", ""); + String dirname = dir.getPath().replaceAll("/+$", ""); if (!checkedDirs.contains(dirname)) { arrDirnames.add(dirname); } @@ -923,7 +927,7 @@ public SimpleEntry> getSubmodulesUrls( } else { uriNormBasename = uriNorm.substring(sep + 1); } - uriNormBasename = uriNormBasename.replaceAll(".git$", ""); + uriNormBasename = uriNormBasename.replaceAll("\\.git$", ""); if (uriNormBasename.equals(needleNormBasename)) { resultFiltered1.add(subEntry); @@ -940,7 +944,7 @@ public SimpleEntry> getSubmodulesUrls( } else { dirBasename = dirName.substring(sep + 1); } - dirBasename = dirBasename.replaceAll(".git$", ""); + dirBasename = dirBasename.replaceAll("\\.git$", ""); if (dirBasename.equals(needleNormBasename)) { resultFiltered2.add(subEntry); @@ -1090,7 +1094,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { needleBasename = url.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".git$", ""); + needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); @@ -1114,7 +1118,7 @@ && getObjectsFile(referenceExpanded + ".git") == null) { } else { needleBasename = urlNormalized.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".git$", ""); + needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); @@ -1192,7 +1196,7 @@ && getObjectsFile(referenceExpanded + ".git") == null) { } else { needleBasename = url.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".git$", ""); + needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); From ad5f34cd68178342bfbdafc1a306692bc7b34e8d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:01:43 +0200 Subject: [PATCH 141/155] Deduplicate getRemoteUrls/getRemotePushUrls via private helper methods Both CliGitAPIImpl and JGitAPIImpl had near-identical implementations of getRemoteUrls() and getRemotePushUrls() differing only in which config key or URIish list accessor they used. The duplication meant bug fixes or improvements had to be applied twice, and the two methods had already started to drift (a comment was present in one but not the other). CliGitAPIImpl: extract getRemoteUrlMap(String configKeySuffix) and delegate both public methods to it, parameterised on ".url=" vs ".pushurl=". JGitAPIImpl: extract collectRemoteUris(Function>) and delegate both public methods via method references RemoteConfig::getURIs and RemoteConfig::getPushURIs. No behaviour change. Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/gitclient/CliGitAPIImpl.java | 37 +++++-------------- .../plugins/gitclient/JGitAPIImpl.java | 26 ++++--------- 2 files changed, 18 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 442549356c..ca98bd14b0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1723,45 +1723,28 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr /** {@inheritDoc} */ @Override public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { - String result = launchCommand("config", "--local", "--list"); - Map uriNames = new HashMap<>(); - for (String line : result.split("\\R+")) { - line = StringUtils.trim(line); - if (!line.startsWith("remote.") || !line.contains(".url=")) { - continue; - } - - String remoteName = StringUtils.substringBetween(line, "remote.", ".url="); - String remoteUri = StringUtils.substringAfter(line, ".url="); - - // If uri String values end up identical, Map only stores one entry - uriNames.put(remoteUri, remoteName); - - try { - URI u = new URI(remoteUri); - uriNames.put(u.toASCIIString(), remoteName); - URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); - uriNames.put(uSafe.toString(), remoteName); - uriNames.put(uSafe.toASCIIString(), remoteName); - } catch (URISyntaxException ue) { - } // ignore, move along - } - return uriNames; + return getRemoteUrlMap(".url="); } /** {@inheritDoc} */ @Override public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { + return getRemoteUrlMap(".pushurl="); + } + + private Map getRemoteUrlMap(String configKeySuffix) throws GitException, InterruptedException { String result = launchCommand("config", "--local", "--list"); Map uriNames = new HashMap<>(); for (String line : result.split("\\R+")) { line = StringUtils.trim(line); - if (!line.startsWith("remote.") || !line.contains(".pushurl=")) { + if (!line.startsWith("remote.") || !line.contains(configKeySuffix)) { continue; } - String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl="); - String remoteUri = StringUtils.substringAfter(line, ".pushurl="); + String remoteName = StringUtils.substringBetween(line, "remote.", configKeySuffix); + String remoteUri = StringUtils.substringAfter(line, configKeySuffix); + + // If uri String values end up identical, Map only stores one entry uriNames.put(remoteUri, remoteName); try { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 5c59513a31..87094afa57 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -55,6 +55,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; @@ -1158,34 +1159,23 @@ public String getRemoteUrl(String name) throws GitException { /** {@inheritDoc} */ @Override public Map getRemoteUrls() throws GitException, InterruptedException { - Map uriNames = new HashMap<>(); - try (Repository repo = getRepository()) { - Config c = repo.getConfig(); - for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { - String remoteName = rc.getName(); - for (URIish u : rc.getURIs()) { - // If uri String values end up identical, Map only stores one entry - uriNames.put(u.toString(), remoteName); - uriNames.put(u.toPrivateString(), remoteName); - uriNames.put(u.toASCIIString(), remoteName); - uriNames.put(u.toPrivateASCIIString(), remoteName); - } - } - } catch (URISyntaxException ue) { - throw new GitException(ue.toString()); - } - return uriNames; + return collectRemoteUris(RemoteConfig::getURIs); } /** {@inheritDoc} */ @Override public Map getRemotePushUrls() throws GitException, InterruptedException { + return collectRemoteUris(RemoteConfig::getPushURIs); + } + + private Map collectRemoteUris(Function> uriSelector) + throws GitException { Map uriNames = new HashMap<>(); try (Repository repo = getRepository()) { Config c = repo.getConfig(); for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { String remoteName = rc.getName(); - for (URIish u : rc.getPushURIs()) { + for (URIish u : uriSelector.apply(rc)) { // If uri String values end up identical, Map only stores one entry uriNames.put(u.toString(), remoteName); uriNames.put(u.toPrivateString(), remoteName); From 9cff320a4b28b54ff3445936c47146c432291abf Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:04:17 +0200 Subject: [PATCH 142/155] Misc code cleanup: simplify, remove dead/duplicate code - isParameterizedReferenceRepository(): collapse 6 independent if/return true blocks into a single return with || operators. - getSubmodulesUrls(): remove large commented-out block (TBD placeholder); replace with a concise TODO comment. Remove the duplicate `isBare = false` assignment inside the else-branch. Remove `result = new LinkedHashSet<>()` before the two exact-match early-return sites; clearing the set there discarded previously cached findings for no benefit. - findParameterizedReferenceRepository(): remove `referencePath = null; // GC` lines; the JVM does not need manual nulling hints and the pattern is noisy. - CliGitAPIImpl / JGitAPIImpl: consolidate the 4-line incremental string construction for the "Parameterized reference path ... replaced with: ..." log message into a single string expression. Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/gitclient/CliGitAPIImpl.java | 6 +- .../plugins/gitclient/JGitAPIImpl.java | 6 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 96 ++----------------- 3 files changed, 12 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index ca98bd14b0..d1b06b1b75 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -858,10 +858,8 @@ public void execute() throws GitException, InterruptedException { } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest - String msg = "Parameterized reference path "; - msg += "'" + reference + "'"; - msg += " replaced with: "; - msg += "'" + referencePath.getPath() + "'"; + String msg = "Parameterized reference path '" + reference + + "' replaced with: '" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 87094afa57..117060714e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1736,10 +1736,8 @@ public void execute() throws GitException { } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest - String msg = "Parameterized reference path "; - msg += "'" + reference + "'"; - msg += " replaced with: "; - msg += "'" + referencePath.getPath() + "'"; + String msg = "Parameterized reference path '" + reference + + "' replaced with: '" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 13f2b3b059..e75c91e088 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -358,31 +358,12 @@ public static Boolean isParameterizedReferenceRepository(String reference) { return false; } - if (reference.endsWith("/${GIT_URL_SHA256}")) { - return true; - } - - if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { - return true; - } - - if (reference.endsWith("/${GIT_URL_BASENAME}")) { - return true; - } - - if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - return true; - } - - if (reference.endsWith("/${GIT_SUBMODULES}")) { - return true; - } - - if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - return true; - } - - return false; + return reference.endsWith("/${GIT_URL_SHA256}") + || reference.endsWith("/${GIT_URL_SHA256_FALLBACK}") + || reference.endsWith("/${GIT_URL_BASENAME}") + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") + || reference.endsWith("/${GIT_SUBMODULES}") + || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}"); } /** There are many ways to spell an URL to the same repository even if @@ -542,7 +523,6 @@ public SimpleEntry> getSubmodulesUrls( } else { // Some other error isBare = false; // At least try to look into submodules... - isBare = false; } LOGGER.log( @@ -652,7 +632,6 @@ public SimpleEntry> getSubmodulesUrls( "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } @@ -696,64 +675,8 @@ public SimpleEntry> getSubmodulesUrls( checkedDirs.add(resultEntry[0]); } - /* - // TBD: Needs a way to list submodules in given workspace and convert - // that into (relative) subdirs, possibly buried some levels deep, for - // cases where the submodule is defined in parent with the needle URL. - // Maybe merge with current "if isBare" below, to optionally seed - // same arrDirnames with different values and check remotes listed - // in those repos. - // If current repo *is NOT* bare - check its submodules - // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory. - // But we can add at higher priority submodule path(s) whose - // basename of the URL matches the needleBasename. And then - // other submodule paths to inspect before arbitrary subdirs. - if (!isBare) { - try { - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // * if we are recursed into a "leaf" project and inspect ITS - // submodules, look at all git tips or even commits, to find - // and inspect all unique (by hash) .gitmodule objects, since - // over time or in different branches a "leaf" project could - // reference different subs? - // getRemoteUrls() => Map - //// arrDirnames.clear(); - - // TODO: Check subdirs that are git workspaces, and remove "|| true" above - //// LinkedHashSet checkedDirs = new LinkedHashSet<>(); - //// for (String[] resultEntry : result) { - //// checkedDirs.add(resultEntry[0]); - //// } - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - } - } catch (Exception e) { - // ignore, go to next slide - } - } - */ + // TODO: If current repo is NOT bare, also check git submodules registered in + // .gitmodules for a faster possible answer (MODNAME.{url,path} mapping). // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, @@ -831,7 +754,6 @@ public SimpleEntry> getSubmodulesUrls( if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri))) { - result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } @@ -1214,7 +1136,6 @@ && getObjectsFile(referenceExpanded + ".git") == null) { if (referenceExpanded != null) { reference = referenceExpanded; - referencePath = null; // GC referencePath = new File(reference); } @@ -1228,7 +1149,6 @@ && getObjectsFile(referenceExpanded + ".git") == null) { // Normalize the URLs with or without .git suffix to // be served by same dir with the refrepo contents reference += ".git"; - referencePath = null; // GC referencePath = new File(reference); } From 15332e595333d66b1f408bb5d085e29f7afdc022 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 16 Jul 2026 13:15:06 +0200 Subject: [PATCH 143/155] LegacyCompatibleGitAPIImpl: getSubmodulesUrls(): with a direct hit, return LinkedHashSet with best result on top Semi-revert the AI change from commit 9cff320a with rationale: > Remove `result = new LinkedHashSet<>()` before the two exact-match > early-return sites; clearing the set there discarded previously > cached findings for no benefit. IIRC the benefit was that the caller needed not look at the other cached candidate locations. Returning a set where the exact-match is listed first, but other findings are not forgotten, may cover both bases. Signed-off-by: Jim Klimov --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e75c91e088..7b958ba364 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -632,8 +632,12 @@ public SimpleEntry> getSubmodulesUrls( "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result.add(new String[] {dirname, uri, uriNorm, remoteName}); - return new SimpleEntry<>(true, result); + // Best hit, caller really only needs to look at this one + // (as the first entry in the returned ordered set) + LinkedHashSet result_best = new LinkedHashSet<>(); // Retain order of insertion + result_best.add(new String[] {dirname, uri, uriNorm, remoteName}); + result_best.addAll(result); + return new SimpleEntry<>(true, result_best); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop @@ -753,9 +757,14 @@ public SimpleEntry> getSubmodulesUrls( + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri))) { - result.add(new String[] {dirname, uri, uriNorm, remoteName}); - return new SimpleEntry<>(true, result); + && (needleNorm.equals(uriNorm) || needle.equals(uri)) + ) { + // Best hit, caller really only needs to look at this one + // (as the first entry in the returned ordered set) + LinkedHashSet result_best = new LinkedHashSet<>(); // Retain order of insertion + result_best.add(new String[] {dirname, uri, uriNorm, remoteName}); + result_best.addAll(result); + return new SimpleEntry<>(true, result_best); } // Cache the finding to return eventually, for each remote: // * absolute dirname of a Git workspace From 8058fa3bcfa242931fd3c021e1d886d3f77b6936 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 16 Jul 2026 13:20:29 +0200 Subject: [PATCH 144/155] LegacyCompatibleGitAPIImpl: getSubmodulesUrls(): refer to commit with dropped commented-away code for not-bare repo handling Signed-off-by: Jim Klimov --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7b958ba364..d5472f61d0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -679,8 +679,10 @@ public SimpleEntry> getSubmodulesUrls( checkedDirs.add(resultEntry[0]); } - // TODO: If current repo is NOT bare, also check git submodules registered in - // .gitmodules for a faster possible answer (MODNAME.{url,path} mapping). + // TODO: If current repo is NOT bare, also check git submodules registered + // in .gitmodules for a faster possible answer (MODNAME.{url,path} mapping). + // See git commit 9cff320a (a couple commits before this line was added) + // where a commented-away exploration of how this might be done was removed. // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, From ea432a9d69edab952a1cac4fa6887442bab94209 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:05:35 +0200 Subject: [PATCH 145/155] Remove debug System.err.println from test helpers in favour of logger; drop duplicate method WorkspaceWithRepo.localMirror(String): remove 7 unconditional System.err.println calls that were added for debugging. They fired on every test run, polluted CI stderr, and are not guarded by any logging level. Instead, add a class-level Logger and convert the 7 unconditional System.err.println calls in localMirror(String) to LOGGER.log(Level.FINE). The messages are still available when debugging (set the logger to FINE), but are suppressed in normal CI runs. GitClientCloneTest: remove 4 System.err.println calls from the two SHA256-parameterized reference repo tests (wsRefrepoBase/wsRefrepo state dumps and clone output dumps). Likewise, add a class-level Logger and convert the 4 System.err.println calls in the two SHA256-parameterized reference repo tests to LOGGER.log(Level.FINE). Also remove the no-arg assertAlternateFilePointsToLocalWorkspaceMirror() overload which was introduced alongside assertAlternateFilePointsToLocalMirror() and is identical in body; the no-arg form is never called, so it is dead code. Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/gitclient/GitClientCloneTest.java | 16 ++++++------ .../plugins/gitclient/WorkspaceWithRepo.java | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b0d249c66c..29d66f7650 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -45,6 +45,8 @@ @MethodSource("gitObjects") class GitClientCloneTest { + private static final Logger LOGGER = Logger.getLogger(GitClientCloneTest.class.getName()); + @RegisterExtension private final GitClientSampleRepoRule repo = new GitClientSampleRepoRule(); @@ -365,7 +367,8 @@ void test_clone_reference_parameterized_sha256() throws Exception { // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", + new Object[]{wsRefrepoBase, wsRefrepo}); testGitClient .clone_() @@ -380,7 +383,7 @@ void test_clone_reference_parameterized_sha256() throws Exception { // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - System.err.println("clone output:\n======\n" + messages + "\n======\n"); + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); assertThat( "Reference repo name-parsing logged in: " + messages @@ -433,7 +436,8 @@ void test_clone_reference_parameterized_sha256_fallback() throws Exception { // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", + new Object[]{wsRefrepoBase, wsRefrepo}); testGitClient .clone_() @@ -448,7 +452,7 @@ void test_clone_reference_parameterized_sha256_fallback() throws Exception { // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - System.err.println("clone output:\n======\n" + messages + "\n======\n"); + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); // Note: we do not expect the closing single quote after wsRefrepoBase // because other tests might pollute our test area, and SHA dir is there @@ -661,10 +665,6 @@ private void assertAlternateFilePointsToLocalMirror() throws Exception { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror() throws Exception { - assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); - } - private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws Exception { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 6e1199a0c4..cb77f8f3a9 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -18,11 +18,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.lib.ObjectId; class WorkspaceWithRepo { + private static final Logger LOGGER = Logger.getLogger(WorkspaceWithRepo.class.getName()); + private GitClient gitClient; private File gitFileDir; private CliGitCommand cliGitCommand; @@ -91,13 +95,15 @@ String localMirror() throws Exception { String localMirror(String cloneDirName) throws Exception { File base = new File(".").getAbsoluteFile(); - System.err.println("=== Beginning to search for cloneDirName='" + cloneDirName + "' from " + base.getPath()); + LOGGER.log(Level.FINE, "Beginning to search for cloneDirName=''{0}'' from {1}", + new Object[]{cloneDirName, base.getPath()}); for (File f = base; f != null; f = f.getParentFile()) { - System.err.println("Looking for 'target' in " + f.getPath()); + LOGGER.log(Level.FINE, "Looking for ''target'' in {0}", f.getPath()); File targetDir = new File(f, "target"); if (targetDir.exists()) { File clone = new File(targetDir, cloneDirName); - System.err.println("Looking for cloneDirName " + cloneDirName + " in " + targetDir.getPath()); + LOGGER.log(Level.FINE, "Looking for cloneDirName {0} in {1}", + new Object[]{cloneDirName, targetDir.getPath()}); if (!clone.exists()) { /* Clone to a temporary directory then move the * temporary directory to the final destination @@ -109,7 +115,8 @@ String localMirror(String cloneDirName) throws Exception { */ Path tempClonePath = Files.createTempDirectory(targetDir.toPath(), "clone-"); String destination = tempClonePath.toFile().getAbsolutePath(); - System.err.println("tempClonePath=" + tempClonePath + " => (FQPN)" + destination); + LOGGER.log(Level.FINE, "tempClonePath={0} => (FQPN){1}", + new Object[]{tempClonePath, destination}); if (isShallow()) { cliGitCommand.run("clone", "--mirror", repoURL, destination); } else { @@ -117,7 +124,7 @@ String localMirror(String cloneDirName) throws Exception { "clone", "--reference", f.getCanonicalPath(), "--mirror", repoURL, destination); } if (!clone.exists()) { // Still a race condition, but a narrow race handled by Files.move() - System.err.println("moving tempClonePath to cloneDirName=" + cloneDirName); + LOGGER.log(Level.FINE, "moving tempClonePath to cloneDirName={0}", cloneDirName); renameAndDeleteDir(tempClonePath, cloneDirName); } else { /* @@ -142,12 +149,14 @@ String localMirror(String cloneDirName) throws Exception { * deleteRecursive() will discard a clone that * 'lost the race'. */ - System.err.println( - "removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); + LOGGER.log(Level.FINE, + "removing extra tempClonePath, we already (race?) have cloneDirName={0}", + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } } else { - System.err.println("FOUND cloneDirName " + cloneDirName + " in " + targetDir.getPath()); + LOGGER.log(Level.FINE, "FOUND cloneDirName {0} in {1}", + new Object[]{cloneDirName, targetDir.getPath()}); } // Strip away the "/./" in "...git-client-plugin/./target/..." return Paths.get(clone.getPath()).normalize().toString(); From 82288c5d40854a78cbc9162361fa2e7b3e7ab77c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:10:43 +0200 Subject: [PATCH 146/155] Remove commented-out dead code from getSubmodulesUrls() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commented-out early-return block had a note suggesting it might "overload the flag's meaning" — it was disabled and the note itself acknowledged the idea was not fully thought through. Remove it to keep the method readable. --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d5472f61d0..464fc18733 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -824,14 +824,6 @@ public SimpleEntry> getSubmodulesUrls( // results looks like it is related to the needle. LinkedHashSet resultFiltered = new LinkedHashSet<>(); - /* - if (!checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once, - // in the parent dir? - return new SimpleEntry<>(false, resultFiltered); - } - */ - // Separate lists by suggestion priority: // 1) URI basename similarity // 2) Directory basename similarity From 20b2ce88902d278a7caec9e337e55707b9229896 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 15 Jul 2026 22:21:20 +0200 Subject: [PATCH 147/155] Add tests for getRemoteUrls, getRemotePushUrls, newGit, and static util methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @MarkEWaite's Sep 2022 review request for test coverage of the new methods introduced by the PR: - GitClientTest.java: five new parameterized tests (run for git/jgit/jgitapache) covering getRemoteUrls() with one and two remotes, getRemotePushUrls() with and without an explicit pushurl configured, and newGit() verifying same-class client creation for a different directory. - LegacyCompatibleGitAPIImplStaticTest.java: new test class for the static utility methods that need no git implementation parameter — getObjectsFile() for null/missing/normal-workspace/bare-repo/submodule-pointer inputs, isParameterizedReferenceRepository() for all six recognized magic suffixes and negative cases, and normalizeGitUrl() for trailing-slash stripping, .git suffix handling, scheme inference (file://, ssh://), and URL canonicalization. --- .../plugins/gitclient/GitClientTest.java | 50 ++++ .../LegacyCompatibleGitAPIImplStaticTest.java | 279 ++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java index 50942159c0..e3cf28a226 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java @@ -1016,6 +1016,56 @@ void testAddRemoteUrl() throws Exception { assertEquals(upstreamRepoURL, gitClient.getRemoteUrl("upstream")); } + @Test + void testGetRemoteUrls() throws Exception { + Map remoteUrls = gitClient.getRemoteUrls(); + assertNotNull(remoteUrls, "getRemoteUrls() returned null"); + assertThat(remoteUrls, hasEntry(srcRepoDir.getAbsolutePath(), "origin")); + } + + @Test + void testGetRemoteUrlsMultipleRemotes() throws Exception { + gitClient.addRemoteUrl("upstream", upstreamRepoURL); + Map remoteUrls = gitClient.getRemoteUrls(); + assertNotNull(remoteUrls, "getRemoteUrls() returned null"); + assertThat(remoteUrls, hasEntry(srcRepoDir.getAbsolutePath(), "origin")); + assertThat(remoteUrls.values(), hasItem("upstream")); + } + + @Test + void testGetRemotePushUrls_noPushUrl() throws Exception { + // No pushurl configured — implementations return null or empty map + Map pushUrls = gitClient.getRemotePushUrls(); + assertTrue( + pushUrls == null || pushUrls.isEmpty(), + "Expected null or empty map when no pushurl is configured, got: " + pushUrls); + } + + @Test + void testGetRemotePushUrls_withPushUrl() throws Exception { + String pushUrl = "git@github.com:example/repo.git"; + CliGitCommand gitCmd = new CliGitCommand(gitClient); + gitCmd.run("config", "remote.origin.pushurl", pushUrl); + Map pushUrls = gitClient.getRemotePushUrls(); + assertNotNull(pushUrls, "getRemotePushUrls() returned null after configuring pushurl"); + assertThat("origin remote should appear in push URL map", pushUrls.values(), hasItem("origin")); + } + + @Test + void testNewGit() throws Exception { + File otherDir = newFolder(tempFolder, "junit-newgit-" + System.nanoTime()); + GitClient otherClient = gitClient.newGit(otherDir.getAbsolutePath()); + assertNotNull(otherClient, "newGit() returned null"); + assertNotSame(gitClient, otherClient, "newGit() returned the same instance"); + assertThat( + "newGit() should return a client of the same implementation class", + otherClient.getClass(), + is(gitClient.getClass())); + // Verify the new client operates on the specified directory + otherClient.init(); + assertTrue(new File(otherDir, ".git").isDirectory(), "newGit() client did not init a git repo"); + } + @Test void testAutocreateFailsOnMultipleMatchingOrigins() throws Exception { File repoRootTemp = newFolder(tempFolder, "junit-" + System.nanoTime()); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java new file mode 100644 index 0000000000..1ee838e4c7 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java @@ -0,0 +1,279 @@ +package org.jenkinsci.plugins.gitclient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import hudson.EnvVars; +import hudson.model.TaskListener; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Unit tests for the static/utility methods of {@link LegacyCompatibleGitAPIImpl}: + * {@code getObjectsFile()}, {@code isParameterizedReferenceRepository()}, + * and {@code normalizeGitUrl()}. + */ +class LegacyCompatibleGitAPIImplStaticTest { + + @TempDir + private File tempDir; + + private File repoDir; + + @BeforeEach + void setUp() throws Exception { + repoDir = newFolder(tempDir, "repo"); + } + + // ----------------------------------------------------------------------- + // getObjectsFile() + // ----------------------------------------------------------------------- + + @Test + void getObjectsFile_nullString_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile((String) null)); + } + + @Test + void getObjectsFile_emptyString_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile("")); + } + + @Test + void getObjectsFile_nullFile_returnsNull() { + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile((File) null)); + } + + @Test + void getObjectsFile_nonExistentPath_returnsNull() { + File missing = new File(tempDir, "does-not-exist"); + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile(missing)); + } + + @Test + void getObjectsFile_regularDirectory_returnsNull() { + // A plain directory without any git metadata should return null + assertNull(LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir)); + } + + @Test + void getObjectsFile_normalWorkspace_returnsObjectsDir() throws Exception { + // A standard (non-bare) git repo has .git/objects/ + Git.with(TaskListener.NULL, new EnvVars()) + .in(repoDir) + .using("git") + .getClient() + .init(); + File expected = new File(repoDir, ".git/objects"); + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for a normal workspace"); + assertThat(result.getAbsolutePath(), is(expected.getAbsolutePath())); + assertTrue(result.isDirectory(), "objects dir should be a directory"); + } + + @Test + void getObjectsFile_bareRepository_returnsObjectsDir() throws Exception { + // A bare repo has objects/ directly in the repo root + Git.with(TaskListener.NULL, new EnvVars()) + .in(repoDir) + .using("git") + .getClient() + .init_() + .workspace(repoDir.getAbsolutePath()) + .bare(true) + .execute(); + File expected = new File(repoDir, "objects"); + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for a bare repo"); + assertThat(result.getAbsolutePath(), is(expected.getAbsolutePath())); + assertTrue(result.isDirectory(), "objects dir should be a directory"); + } + + @Test + void getObjectsFile_submoduleGitPointerFile_returnsObjectsDir() throws Exception { + // Submodule workspaces have a .git FILE (not dir) containing "gitdir: " + // The real git metadata lives at the path referenced by that file. + File realGitDir = newFolder(tempDir, "real-gitdir"); + File realObjects = new File(realGitDir, "objects"); + assertTrue(realObjects.mkdir(), "Failed to create fake objects dir"); + + // Write a .git pointer file in repoDir + File gitPointer = new File(repoDir, ".git"); + String relativeGitdirPath = "../real-gitdir"; + Files.writeString( + gitPointer.toPath(), + "gitdir: " + relativeGitdirPath + "\n", + StandardCharsets.UTF_8); + + File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); + assertNotNull(result, "getObjectsFile() returned null for submodule pointer workspace"); + assertTrue(result.isDirectory(), "Expected objects dir to be a real directory"); + // Canonical path resolves the ".." in the relative gitdir reference + assertThat(result.getCanonicalPath(), is(realObjects.getCanonicalPath())); + } + + // ----------------------------------------------------------------------- + // isParameterizedReferenceRepository() + // ----------------------------------------------------------------------- + + @Test + void isParameterizedReferenceRepository_null_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository((String) null)); + } + + @Test + void isParameterizedReferenceRepository_emptyString_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("")); + } + + @Test + void isParameterizedReferenceRepository_plainPath_returnsFalse() { + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/repos")); + } + + @Test + void isParameterizedReferenceRepository_sha256Suffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_SHA256}")); + } + + @Test + void isParameterizedReferenceRepository_sha256FallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_SHA256_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_basenameSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_BASENAME}")); + } + + @Test + void isParameterizedReferenceRepository_basenameFallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_BASENAME_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_submodulesSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_SUBMODULES}")); + } + + @Test + void isParameterizedReferenceRepository_submodulesFallbackSuffix_returnsTrue() { + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_SUBMODULES_FALLBACK}")); + } + + @Test + void isParameterizedReferenceRepository_tokenInMiddle_returnsFalse() { + // Token must be at the END of the path; mid-path token is not recognized + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( + "/var/cache/git/${GIT_URL_SHA256}/extra")); + } + + @Test + void isParameterizedReferenceRepository_fileOverload_delegatesToStringVersion() { + File parameterized = new File("/var/cache/git/${GIT_URL_BASENAME}"); + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(parameterized)); + File plain = new File("/var/cache/git/repos"); + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(plain)); + } + + // ----------------------------------------------------------------------- + // normalizeGitUrl() + // ----------------------------------------------------------------------- + + @Test + void normalizeGitUrl_trailingSlashes_areStripped() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo///", false); + assertThat(result, not(endsWith("/"))); + } + + @Test + void normalizeGitUrl_dotGitSuffix_isStripped() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git", false); + assertThat(result, not(endsWith(".git"))); + } + + @Test + void normalizeGitUrl_mixedCaseDotGit_lowercasedButNotStripped() { + // The regex strips only exact lowercase ".git"; ".Git" is lowercased to ".git" but stays + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.Git", false); + assertThat(result, endsWith(".git")); + } + + @Test + void normalizeGitUrl_httpsUrl_isLowercased() { + String result = + LegacyCompatibleGitAPIImpl.normalizeGitUrl("HTTPS://GitHub.com/Example/Repo.git", false); + assertThat(result, startsWith("https://")); + assertThat(result, containsString("github.com")); + } + + @Test + void normalizeGitUrl_absoluteLocalPath_getsFileScheme() { + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo", false); + assertThat(result, startsWith("file://")); + } + + @Test + void normalizeGitUrl_absoluteLocalPath_dotGitStripped() { + String withGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo.git", false); + String withoutGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("/var/cache/git/repo", false); + assertThat(withGit, is(withoutGit)); + } + + @Test + void normalizeGitUrl_relativeLocalPath_getsFileScheme() { + // A path starting with "./" is treated as a local relative path + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("./some/local/repo", false); + assertThat(result, startsWith("file://")); + } + + @Test + void normalizeGitUrl_bareHostname_getsSshScheme() { + // A string without schema, not starting with / or ., is treated as an ssh host + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("git@github.com:example/repo.git", false); + assertThat(result, startsWith("ssh://")); + } + + @Test + void normalizeGitUrl_sameUrlDifferentSuffixes_normalizeToSame() { + String base = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo", false); + String withGit = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git", false); + String withSlash = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo/", false); + String withBoth = LegacyCompatibleGitAPIImpl.normalizeGitUrl("https://github.com/example/repo.git/", false); + assertThat(withGit, is(base)); + assertThat(withSlash, is(base)); + assertThat(withBoth, is(base)); + } + + // ----------------------------------------------------------------------- + // Helper + // ----------------------------------------------------------------------- + + private static File newFolder(File root, String... subDirs) throws IOException { + String subFolder = String.join("/", subDirs); + File result = new File(root, subFolder); + if (!result.mkdirs()) { + throw new IOException("Couldn't create folders " + result); + } + return result; + } +} From 66d06f40c962b081048517bbbbc95b00bd24e1fe Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 16 Jul 2026 12:28:06 +0200 Subject: [PATCH 148/155] Fix two test failures from platform and wrapper-class assumptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit testNewGit: The assertion comparing getClass() of the gitClient field (which is a hudson.plugins.git.GitAPI wrapper) to the class returned by newGit() (which is the unwrapped CliGitAPIImpl) always fails. Replace the class-equality check with the functionally meaningful assertion: the returned client can actually init a git repo in the specified directory. isParameterizedReferenceRepository_fileOverload: On Windows, File.getPath() uses backslash separators while the parameterized-suffix checks use forward slashes, so new File("/.../${GIT_URL_BASENAME}").getPath() does not end with "/${GIT_URL_BASENAME}". Restrict the File-overload test to platform-independent cases (null → false, non-parameterized → false). --- .../org/jenkinsci/plugins/gitclient/GitClientTest.java | 6 +----- .../LegacyCompatibleGitAPIImplStaticTest.java | 10 ++++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java index e3cf28a226..029c762bc7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientTest.java @@ -1057,11 +1057,7 @@ void testNewGit() throws Exception { GitClient otherClient = gitClient.newGit(otherDir.getAbsolutePath()); assertNotNull(otherClient, "newGit() returned null"); assertNotSame(gitClient, otherClient, "newGit() returned the same instance"); - assertThat( - "newGit() should return a client of the same implementation class", - otherClient.getClass(), - is(gitClient.getClass())); - // Verify the new client operates on the specified directory + // Verify the new client is a usable GitClient, not just a non-null reference otherClient.init(); assertTrue(new File(otherDir, ".git").isDirectory(), "newGit() client did not init a git repo"); } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java index 1ee838e4c7..7a6cc42711 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java @@ -189,10 +189,12 @@ void isParameterizedReferenceRepository_tokenInMiddle_returnsFalse() { @Test void isParameterizedReferenceRepository_fileOverload_delegatesToStringVersion() { - File parameterized = new File("/var/cache/git/${GIT_URL_BASENAME}"); - assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(parameterized)); - File plain = new File("/var/cache/git/repos"); - assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(plain)); + // The File overload delegates to the String overload via file.getPath(). + // On Windows, getPath() uses backslashes while the checks use forward slashes, + // so we only assert the behaviors that are platform-independent: + // null File → false, and non-parameterized File → false. + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository((File) null)); + assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(new File("/var/cache/git/repos"))); } // ----------------------------------------------------------------------- From 1e1d4f8680161b060f992d001b4d1085ecfc2feb Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 20 Jul 2026 21:23:32 +0200 Subject: [PATCH 149/155] Fix assertAlternateFilePointsToLocalMirror using field instead of param; drop redundant public on interface methods GitClientCloneTest.assertAlternateFilePointsToLocalMirror(): the method takes a _testGitDir parameter but line 708 read the alternates file using the class field testGitDir instead, silently ignoring the parameter. Any caller passing a different directory would assert the wrong file content. Fix: use new File(_testGitDir).toPath() so the parameter is honoured. GitClient.java: interface methods are implicitly public; the explicit 'public' modifier on getRemoteUrls() and getRemotePushUrls() was inconsistent with every other method declaration in the interface. Remove the redundant modifier. --- src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java | 4 ++-- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index cb94f44528..f0851bb058 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -230,7 +230,7 @@ public interface GitClient { * repository/workspace, and values represent names. * There may be several URIs corresponding to same name. */ - public Map getRemoteUrls() throws GitException, InterruptedException; + Map getRemoteUrls() throws GitException, InterruptedException; /** * getRemotePushUrls. @@ -241,7 +241,7 @@ public interface GitClient { * repository/workspace, and values represent names. * There may be several URIs corresponding to same name. */ - public Map getRemotePushUrls() throws GitException, InterruptedException; + Map getRemotePushUrls() throws GitException, InterruptedException; /** * For a given repository, set a remote's URL diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 29d66f7650..4f325d1783 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -705,7 +705,7 @@ private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _ new File(_testGitDir, alternates), is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; - final String actualContent = Files.readString(testGitDir.toPath().resolve(alternates), StandardCharsets.UTF_8); + final String actualContent = Files.readString(new File(_testGitDir).toPath().resolve(alternates), StandardCharsets.UTF_8); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); From f01c199e71a4862bc76bd78066cf68a67fc9f046 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 20 Jul 2026 21:25:50 +0200 Subject: [PATCH 150/155] Fix LinkedHashSet not deduplicating: switch to LinkedHashSet> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String[] uses identity-based equals/hashCode, so a LinkedHashSet never actually removes duplicate entries — every new String[] {} is a distinct object. The comment at the merge point ("Hopefully the Set should deduplicate entries if something matched twice") was therefore never true. Change the element type from String[] to List throughout getSubmodulesUrls() and its callers. List.equals() / List.hashCode() are content-based, so the LinkedHashSet now correctly deduplicates entries with the same (dirname, url, urlNormalized, remoteName) tuple. Mechanical changes: - LinkedHashSet → LinkedHashSet> (10 sites) - new String[] {a,b,c,d} → List.of(a,b,c,d) (4 sites) - entry[N] → entry.get(N) (9 sites) - for (String[] ...) → for (List ...) (4 sites) - Javadoc updated: [N] → get(N) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 464fc18733..4ca2d764dc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -442,16 +442,16 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * * @return an AbstractMap.SimpleEntry, containing a Boolean to denote * an exact match (or lack thereof) for the needle (if searched for), - * and a Set of (unique) String arrays, representing: - * [0] directory of nested submodule (relative to current workspace root) + * and a Set of (unique) String lists, each with four elements: + * get(0) directory of nested submodule (relative to current workspace root) * The current workspace would be listed as directory "" and consumers * should check these entries last if they care for most-specific hits * with smaller-scope reference repositories. - * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several + * get(1) url as returned by getRemoteUrls() - fetch URLs, maybe several * entries per remote - * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames + * get(2) urlNormalized from normalizeGitUrl(url, true) (local pathnames * fully qualified) - * [3] remoteName as defined in that nested submodule + * get(3) remoteName as defined in that nested submodule * * If the returned SimpleEntry has the Boolean flag as False but also * a Set which is not empty, and a search for "needle" was requested, @@ -476,12 +476,12 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * when recursing, since this directory was checked already * as part of parent directory inspection. */ - public SimpleEntry> getSubmodulesUrls( + public SimpleEntry>> getSubmodulesUrls( String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. - LinkedHashSet result = new LinkedHashSet<>(); // Retain order of insertion + LinkedHashSet> result = new LinkedHashSet<>(); // Retain order of insertion File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); @@ -634,15 +634,15 @@ public SimpleEntry> getSubmodulesUrls( if (needleNorm.equals(uriNorm) || needle.equals(uri)) { // Best hit, caller really only needs to look at this one // (as the first entry in the returned ordered set) - LinkedHashSet result_best = new LinkedHashSet<>(); // Retain order of insertion - result_best.add(new String[] {dirname, uri, uriNorm, remoteName}); + LinkedHashSet> result_best = new LinkedHashSet<>(); // Retain order of insertion + result_best.add(List.of(dirname, uri, uriNorm, remoteName)); result_best.addAll(result); return new SimpleEntry<>(true, result_best); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[] {dirname, uri, uriNorm, remoteName}); + result.add(List.of(dirname, uri, uriNorm, remoteName)); } } catch (Throwable t) { // ignore, go to next slide @@ -675,8 +675,8 @@ public SimpleEntry> getSubmodulesUrls( // Track where we have looked already; note that values in result[] // (if any from needle-search above) are absolute pathnames LinkedHashSet checkedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); + for (List resultEntry : result) { + checkedDirs.add(resultEntry.get(0)); } // TODO: If current repo is NOT bare, also check git submodules registered @@ -763,8 +763,8 @@ public SimpleEntry> getSubmodulesUrls( ) { // Best hit, caller really only needs to look at this one // (as the first entry in the returned ordered set) - LinkedHashSet result_best = new LinkedHashSet<>(); // Retain order of insertion - result_best.add(new String[] {dirname, uri, uriNorm, remoteName}); + LinkedHashSet> result_best = new LinkedHashSet<>(); // Retain order of insertion + result_best.add(List.of(dirname, uri, uriNorm, remoteName)); result_best.addAll(result); return new SimpleEntry<>(true, result_best); } @@ -773,7 +773,7 @@ public SimpleEntry> getSubmodulesUrls( // * original remote URI from that workspace's config // * normalized remote URI // * name of the remote from that workspace's config ("origin" etc) - result.add(new String[] {dirname, uri, uriNorm, remoteName}); + result.add(List.of(dirname, uri, uriNorm, remoteName)); } } catch (Throwable t) { // ignore, go to next slide @@ -789,9 +789,9 @@ public SimpleEntry> getSubmodulesUrls( // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); + SimpleEntry>> subEntriesRet = getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); - LinkedHashSet subEntries = subEntriesRet.getValue(); + LinkedHashSet> subEntries = subEntriesRet.getValue(); LOGGER.log( Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " @@ -802,7 +802,7 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname - + "': " + subEntries.iterator().next()[0]); + + "': " + subEntries.iterator().next().get(0)); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -822,24 +822,24 @@ public SimpleEntry> getSubmodulesUrls( // Handle suggestions (not-exact matches) if something from // results looks like it is related to the needle. - LinkedHashSet resultFiltered = new LinkedHashSet<>(); + LinkedHashSet> resultFiltered = new LinkedHashSet<>(); // Separate lists by suggestion priority: // 1) URI basename similarity // 2) Directory basename similarity - LinkedHashSet resultFiltered1 = new LinkedHashSet<>(); - LinkedHashSet resultFiltered2 = new LinkedHashSet<>(); + LinkedHashSet> resultFiltered1 = new LinkedHashSet<>(); + LinkedHashSet> resultFiltered2 = new LinkedHashSet<>(); LinkedHashSet suggestedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); + for (List resultEntry : result) { + checkedDirs.add(resultEntry.get(0)); } - for (String[] subEntry : result) { + for (List subEntry : result) { // Iterating to filter suggestions in order of original // directory-walk prioritization under current reference - String dirName = subEntry[0]; - String uriNorm = subEntry[2]; + String dirName = subEntry.get(0); + String uriNorm = subEntry.get(2); Integer sep; String uriNormBasename; String dirBasename; @@ -894,7 +894,7 @@ public SimpleEntry> getSubmodulesUrls( /** See above. With null needle, returns all data we can find under the * referenceBaseDir tree (can take a while) for the caller to parse */ - public SimpleEntry> getSubmodulesUrls(String referenceBaseDir) { + public SimpleEntry>> getSubmodulesUrls(String referenceBaseDir) { return getSubmodulesUrls(referenceBaseDir, null, true); } /* Do we need a completely parameter-less variant to look under current @@ -1077,10 +1077,10 @@ && getObjectsFile(referenceExpanded + ".git") == null) { // Note: we pass the unmodified "url" value here, the routine // differentiates original spelling vs. normalization while // looking for its needle in the haystack. - SimpleEntry> subEntriesRet = + SimpleEntry>> subEntriesRet = getSubmodulesUrls(referenceBaseDir, url, true); Boolean subEntriesExactMatched = subEntriesRet.getKey(); - LinkedHashSet subEntries = subEntriesRet.getValue(); + LinkedHashSet> subEntries = subEntriesRet.getValue(); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new @@ -1088,15 +1088,15 @@ && getObjectsFile(referenceExpanded + ".git") == null) { // If several entries are present after all, iterate until // first existing hit and return the first entry otherwise. if (!subEntriesExactMatched) { // else look at first entry below - for (String[] subEntry : subEntries) { - if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { - referenceExpanded = subEntry[0]; + for (List subEntry : subEntries) { + if (getObjectsFile(subEntry.get(0)) != null || getObjectsFile(subEntry.get(0) + ".git") != null) { + referenceExpanded = subEntry.get(0); break; } } } if (referenceExpanded == null) { - referenceExpanded = subEntries.iterator().next()[0]; + referenceExpanded = subEntries.iterator().next().get(0); } LOGGER.log( Level.FINE, From d1188d7f5a9168be18d7c8c6c1f95994ab5141a7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 22 Jul 2026 22:50:35 +0200 Subject: [PATCH 151/155] LegacyCompatibleGitAPIImplTest: add some test cases around getSubmodulesUrls() and findParameterizedReferenceRepository() [JENKINS-64383] Signed-off-by: Evgeny Klimov Co-Authored-By: Claude Sonnet 4.6 --- .../LegacyCompatibleGitAPIImplStaticTest.java | 2 +- .../LegacyCompatibleGitAPIImplTest.java | 112 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java index 7a6cc42711..07cda3920c 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java @@ -192,7 +192,7 @@ void isParameterizedReferenceRepository_fileOverload_delegatesToStringVersion() // The File overload delegates to the String overload via file.getPath(). // On Windows, getPath() uses backslashes while the checks use forward slashes, // so we only assert the behaviors that are platform-independent: - // null File → false, and non-parameterized File → false. + // null File -> false, and non-parameterized File -> false. assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository((File) null)); assertFalse(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository(new File("/var/cache/git/repos"))); } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java index 04a8e380da..96652f2f94 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java @@ -1,5 +1,9 @@ package org.jenkinsci.plugins.gitclient; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.*; import hudson.model.TaskListener; @@ -12,9 +16,13 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.AbstractMap; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.RemoteConfig; @@ -249,6 +257,110 @@ void testExtractBranchNameFromBranchSpec() { assertEquals("refs/tags/mytag", git.extractBranchNameFromBranchSpec("refs/tags/mytag")); } + // ----------------------------------------------------------------------- + // getSubmodulesUrls() and findParameterizedReferenceRepository() tests [JENKINS-64383] + // NOTE: The name "getSubmodulesUrls" is a bit misleading here, so far. + // It's named for the use case (a fanout reference repo that *might* contain + // submodule-like repos), but the implementation currently looks for + // "any subdir with a git objects dir", not for registered git submodules. + // Code has a "TODO" that is not implemented yet: + // If current repo is NOT bare, also check git submodules + // registered in .gitmodules for a faster possible answer + // ----------------------------------------------------------------------- + + /** + * Creates a subdirectory inside {@code parent}, initializes a git repo + * there, and registers a remote "origin" pointing to {@code remoteUrl}. + */ + private File createSubdirWithRemote(File parent, String dirName, String remoteUrl) throws Exception { + File subDir = newFolder(parent, dirName); + GitClient subGit = Git.with(listener, env).in(subDir).using(gitImpl).getClient(); + subGit.init(); + new CliGitCommand(subGit).run("remote", "add", "origin", remoteUrl); + return subDir; + } + + @Test + void testGetSubmodulesUrls_noNeedle_returnsAllSubdirs() throws Exception { + File base = newFolder(tempFolder, "refrepo-noneedle"); + File repoAlpha = createSubdirWithRemote(base, "alpha-repo", "https://github.com/example/alpha-project.git"); + File repoBeta = createSubdirWithRemote(base, "beta-repo", "https://github.com/example/beta-project.git"); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath()); + + assertFalse(result.getKey(), "No needle: flag should be false"); + LinkedHashSet> entries = result.getValue(); + assertFalse(entries.isEmpty(), "Should find at least one subdir"); + Set foundDirs = + entries.stream().map(e -> new File(e.get(0)).getAbsolutePath()).collect(Collectors.toSet()); + assertThat("alpha-repo should appear in results", foundDirs, hasItem(repoAlpha.getAbsolutePath())); + assertThat("beta-repo should appear in results", foundDirs, hasItem(repoBeta.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_exactNeedleMatch_promotedToFirst() throws Exception { + // "target-basename" dir is named to match the needle's URL basename, so it is + // probed in the prefix-search phase (before any listFiles() walk). It holds a + // *different* URL -> no exact match -> accumulated into result first. + // "aaa-exact" holds the exact needle URL and is found in the general walk that + // follows. The exact-match return promotes it to result_best[0] with all + // previously accumulated entries after it, giving at least 2 entries total. + String needleUrl = "https://github.com/example/target-basename.git"; + File base = newFolder(tempFolder, "refrepo-exactmatch"); + createSubdirWithRemote(base, "target-basename", "https://github.com/other-org/not-the-target.git"); + File repoExact = createSubdirWithRemote(base, "aaa-exact", needleUrl); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Exact URL match: flag should be true"); + LinkedHashSet> entries = result.getValue(); + assertThat("Exact-match entry + previously accumulated entry = at least 2", entries.size(), greaterThanOrEqualTo(2)); + String firstDir = entries.iterator().next().get(0); + assertThat("Exact-match repo is promoted to first position", + new File(firstDir).getAbsolutePath(), is(repoExact.getAbsolutePath())); + } + + @Test + void testFindParameterizedReferenceRepository_submodules_exactMatchPickedAsResult() throws Exception { + // Same two-repo setup: prefix-search accumulates "target-basename" (non-exact), + // then the general walk finds "aaa-exact" (exact). The consuming code uses the + // first entry in the result set, which is the exact match. + String needleUrl = "https://github.com/example/target-basename.git"; + File base = newFolder(tempFolder, "refrepo-find-exact"); + createSubdirWithRemote(base, "target-basename", "https://github.com/other-org/not-the-target.git"); + File repoExact = createSubdirWithRemote(base, "aaa-exact", needleUrl); + String reference = base.getAbsolutePath() + "/${GIT_SUBMODULES}"; + + File result = git.findParameterizedReferenceRepository(reference, needleUrl); + + assertNotNull(result, "Should return a result for an exact URL match"); + assertThat("Exact-match subdir should be returned", + result.getCanonicalPath(), is(repoExact.getCanonicalPath())); + } + + @Test + void testFindParameterizedReferenceRepository_submodules_firstEntryUsedForNonExact() throws Exception { + // "cool-project" dir matches the needle's URL basename and is probed first in + // the prefix-search phase -> ends up first in the ordered suggestion set. + // "another-fork" is found in the general walk with the same URL basename. + // With no exact match, findParameterizedReferenceRepository iterates the set + // and returns the first entry that is an existing git repo -> "cool-project". + String needleUrl = "https://github.com/example/cool-project.git"; + File base = newFolder(tempFolder, "refrepo-find-nonexact"); + File repoFirst = + createSubdirWithRemote(base, "cool-project", "https://github.com/fork-a/cool-project.git"); + createSubdirWithRemote(base, "another-fork", "https://github.com/fork-b/cool-project.git"); + String reference = base.getAbsolutePath() + "/${GIT_SUBMODULES}"; + + File result = git.findParameterizedReferenceRepository(reference, needleUrl); + + assertNotNull(result, "Should return a result when non-exact URL-basename matches exist"); + assertThat("First entry from the ordered suggestion set should be returned", + result.getCanonicalPath(), is(repoFirst.getCanonicalPath())); + } + private static File newFolder(File root, String... subDirs) throws Exception { String subFolder = String.join("/", subDirs); File result = new File(root, subFolder); From 72c54a111d48a21dd6442b9c6edb5d8bffa4e2ec Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 22 Jul 2026 23:40:59 +0200 Subject: [PATCH 152/155] Run mvn spotless:apply Signed-off-by: Jim Klimov --- .../plugins/gitclient/CliGitAPIImpl.java | 4 ++-- .../plugins/gitclient/JGitAPIImpl.java | 4 ++-- .../gitclient/LegacyCompatibleGitAPIImpl.java | 18 ++++++++------ .../plugins/gitclient/GitClientCloneTest.java | 9 ++++--- .../LegacyCompatibleGitAPIImplStaticTest.java | 17 ++++--------- .../LegacyCompatibleGitAPIImplTest.java | 24 ++++++++++++------- .../plugins/gitclient/WorkspaceWithRepo.java | 21 +++++++++------- 7 files changed, 51 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index d1b06b1b75..c650138c4a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -858,8 +858,8 @@ public void execute() throws GitException, InterruptedException { } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest - String msg = "Parameterized reference path '" + reference - + "' replaced with: '" + referencePath.getPath() + "'"; + String msg = "Parameterized reference path '" + reference + "' replaced with: '" + + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 117060714e..dc765990bc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1736,8 +1736,8 @@ public void execute() throws GitException { } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest - String msg = "Parameterized reference path '" + reference - + "' replaced with: '" + referencePath.getPath() + "'"; + String msg = "Parameterized reference path '" + reference + "' replaced with: '" + + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 4ca2d764dc..38383024f3 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -374,7 +374,8 @@ public static Boolean isParameterizedReferenceRepository(String reference) { value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { - String urlNormalized = url.replaceAll("/+$", "").replaceAll("\\.git$", "").toLowerCase(Locale.ENGLISH); + String urlNormalized = + url.replaceAll("/+$", "").replaceAll("\\.git$", "").toLowerCase(Locale.ENGLISH); if (!url.contains("://")) { if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname @@ -634,7 +635,8 @@ public SimpleEntry>> getSubmodulesUrls( if (needleNorm.equals(uriNorm) || needle.equals(uri)) { // Best hit, caller really only needs to look at this one // (as the first entry in the returned ordered set) - LinkedHashSet> result_best = new LinkedHashSet<>(); // Retain order of insertion + LinkedHashSet> result_best = + new LinkedHashSet<>(); // Retain order of insertion result_best.add(List.of(dirname, uri, uriNorm, remoteName)); result_best.addAll(result); return new SimpleEntry<>(true, result_best); @@ -759,11 +761,11 @@ public SimpleEntry>> getSubmodulesUrls( + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri)) - ) { + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { // Best hit, caller really only needs to look at this one // (as the first entry in the returned ordered set) - LinkedHashSet> result_best = new LinkedHashSet<>(); // Retain order of insertion + LinkedHashSet> result_best = + new LinkedHashSet<>(); // Retain order of insertion result_best.add(List.of(dirname, uri, uriNorm, remoteName)); result_best.addAll(result); return new SimpleEntry<>(true, result_best); @@ -789,7 +791,8 @@ public SimpleEntry>> getSubmodulesUrls( // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry>> subEntriesRet = getSubmodulesUrls(dirname, needle, false); + SimpleEntry>> subEntriesRet = + getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet> subEntries = subEntriesRet.getValue(); LOGGER.log( @@ -1089,7 +1092,8 @@ && getObjectsFile(referenceExpanded + ".git") == null) { // first existing hit and return the first entry otherwise. if (!subEntriesExactMatched) { // else look at first entry below for (List subEntry : subEntries) { - if (getObjectsFile(subEntry.get(0)) != null || getObjectsFile(subEntry.get(0) + ".git") != null) { + if (getObjectsFile(subEntry.get(0)) != null + || getObjectsFile(subEntry.get(0) + ".git") != null) { referenceExpanded = subEntry.get(0); break; } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 4f325d1783..90125b296e 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -367,8 +367,7 @@ void test_clone_reference_parameterized_sha256() throws Exception { // the provided string, as we check in log below } - LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", - new Object[]{wsRefrepoBase, wsRefrepo}); + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); testGitClient .clone_() @@ -436,8 +435,7 @@ void test_clone_reference_parameterized_sha256_fallback() throws Exception { // the provided string, as we check in log below } - LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", - new Object[]{wsRefrepoBase, wsRefrepo}); + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); testGitClient .clone_() @@ -705,7 +703,8 @@ private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _ new File(_testGitDir, alternates), is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; - final String actualContent = Files.readString(new File(_testGitDir).toPath().resolve(alternates), StandardCharsets.UTF_8); + final String actualContent = + Files.readString(new File(_testGitDir).toPath().resolve(alternates), StandardCharsets.UTF_8); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java index 07cda3920c..b1350dac10 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplStaticTest.java @@ -113,10 +113,7 @@ void getObjectsFile_submoduleGitPointerFile_returnsObjectsDir() throws Exception // Write a .git pointer file in repoDir File gitPointer = new File(repoDir, ".git"); String relativeGitdirPath = "../real-gitdir"; - Files.writeString( - gitPointer.toPath(), - "gitdir: " + relativeGitdirPath + "\n", - StandardCharsets.UTF_8); + Files.writeString(gitPointer.toPath(), "gitdir: " + relativeGitdirPath + "\n", StandardCharsets.UTF_8); File result = LegacyCompatibleGitAPIImpl.getObjectsFile(repoDir); assertNotNull(result, "getObjectsFile() returned null for submodule pointer workspace"); @@ -146,8 +143,7 @@ void isParameterizedReferenceRepository_plainPath_returnsFalse() { @Test void isParameterizedReferenceRepository_sha256Suffix_returnsTrue() { - assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( - "/var/cache/git/${GIT_URL_SHA256}")); + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_URL_SHA256}")); } @Test @@ -158,8 +154,7 @@ void isParameterizedReferenceRepository_sha256FallbackSuffix_returnsTrue() { @Test void isParameterizedReferenceRepository_basenameSuffix_returnsTrue() { - assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( - "/var/cache/git/${GIT_URL_BASENAME}")); + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_URL_BASENAME}")); } @Test @@ -170,8 +165,7 @@ void isParameterizedReferenceRepository_basenameFallbackSuffix_returnsTrue() { @Test void isParameterizedReferenceRepository_submodulesSuffix_returnsTrue() { - assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository( - "/var/cache/git/${GIT_SUBMODULES}")); + assertTrue(LegacyCompatibleGitAPIImpl.isParameterizedReferenceRepository("/var/cache/git/${GIT_SUBMODULES}")); } @Test @@ -222,8 +216,7 @@ void normalizeGitUrl_mixedCaseDotGit_lowercasedButNotStripped() { @Test void normalizeGitUrl_httpsUrl_isLowercased() { - String result = - LegacyCompatibleGitAPIImpl.normalizeGitUrl("HTTPS://GitHub.com/Example/Repo.git", false); + String result = LegacyCompatibleGitAPIImpl.normalizeGitUrl("HTTPS://GitHub.com/Example/Repo.git", false); assertThat(result, startsWith("https://")); assertThat(result, containsString("github.com")); } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java index 96652f2f94..c6c37461e1 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java @@ -316,10 +316,15 @@ void testGetSubmodulesUrls_exactNeedleMatch_promotedToFirst() throws Exception { assertTrue(result.getKey(), "Exact URL match: flag should be true"); LinkedHashSet> entries = result.getValue(); - assertThat("Exact-match entry + previously accumulated entry = at least 2", entries.size(), greaterThanOrEqualTo(2)); + assertThat( + "Exact-match entry + previously accumulated entry = at least 2", + entries.size(), + greaterThanOrEqualTo(2)); String firstDir = entries.iterator().next().get(0); - assertThat("Exact-match repo is promoted to first position", - new File(firstDir).getAbsolutePath(), is(repoExact.getAbsolutePath())); + assertThat( + "Exact-match repo is promoted to first position", + new File(firstDir).getAbsolutePath(), + is(repoExact.getAbsolutePath())); } @Test @@ -336,8 +341,8 @@ void testFindParameterizedReferenceRepository_submodules_exactMatchPickedAsResul File result = git.findParameterizedReferenceRepository(reference, needleUrl); assertNotNull(result, "Should return a result for an exact URL match"); - assertThat("Exact-match subdir should be returned", - result.getCanonicalPath(), is(repoExact.getCanonicalPath())); + assertThat( + "Exact-match subdir should be returned", result.getCanonicalPath(), is(repoExact.getCanonicalPath())); } @Test @@ -349,16 +354,17 @@ void testFindParameterizedReferenceRepository_submodules_firstEntryUsedForNonExa // and returns the first entry that is an existing git repo -> "cool-project". String needleUrl = "https://github.com/example/cool-project.git"; File base = newFolder(tempFolder, "refrepo-find-nonexact"); - File repoFirst = - createSubdirWithRemote(base, "cool-project", "https://github.com/fork-a/cool-project.git"); + File repoFirst = createSubdirWithRemote(base, "cool-project", "https://github.com/fork-a/cool-project.git"); createSubdirWithRemote(base, "another-fork", "https://github.com/fork-b/cool-project.git"); String reference = base.getAbsolutePath() + "/${GIT_SUBMODULES}"; File result = git.findParameterizedReferenceRepository(reference, needleUrl); assertNotNull(result, "Should return a result when non-exact URL-basename matches exist"); - assertThat("First entry from the ordered suggestion set should be returned", - result.getCanonicalPath(), is(repoFirst.getCanonicalPath())); + assertThat( + "First entry from the ordered suggestion set should be returned", + result.getCanonicalPath(), + is(repoFirst.getCanonicalPath())); } private static File newFolder(File root, String... subDirs) throws Exception { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index cb77f8f3a9..3bee8d0029 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -95,15 +95,17 @@ String localMirror() throws Exception { String localMirror(String cloneDirName) throws Exception { File base = new File(".").getAbsoluteFile(); - LOGGER.log(Level.FINE, "Beginning to search for cloneDirName=''{0}'' from {1}", - new Object[]{cloneDirName, base.getPath()}); + LOGGER.log(Level.FINE, "Beginning to search for cloneDirName=''{0}'' from {1}", new Object[] { + cloneDirName, base.getPath() + }); for (File f = base; f != null; f = f.getParentFile()) { LOGGER.log(Level.FINE, "Looking for ''target'' in {0}", f.getPath()); File targetDir = new File(f, "target"); if (targetDir.exists()) { File clone = new File(targetDir, cloneDirName); - LOGGER.log(Level.FINE, "Looking for cloneDirName {0} in {1}", - new Object[]{cloneDirName, targetDir.getPath()}); + LOGGER.log(Level.FINE, "Looking for cloneDirName {0} in {1}", new Object[] { + cloneDirName, targetDir.getPath() + }); if (!clone.exists()) { /* Clone to a temporary directory then move the * temporary directory to the final destination @@ -115,8 +117,7 @@ String localMirror(String cloneDirName) throws Exception { */ Path tempClonePath = Files.createTempDirectory(targetDir.toPath(), "clone-"); String destination = tempClonePath.toFile().getAbsolutePath(); - LOGGER.log(Level.FINE, "tempClonePath={0} => (FQPN){1}", - new Object[]{tempClonePath, destination}); + LOGGER.log(Level.FINE, "tempClonePath={0} => (FQPN){1}", new Object[] {tempClonePath, destination}); if (isShallow()) { cliGitCommand.run("clone", "--mirror", repoURL, destination); } else { @@ -149,14 +150,16 @@ String localMirror(String cloneDirName) throws Exception { * deleteRecursive() will discard a clone that * 'lost the race'. */ - LOGGER.log(Level.FINE, + LOGGER.log( + Level.FINE, "removing extra tempClonePath, we already (race?) have cloneDirName={0}", cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } } else { - LOGGER.log(Level.FINE, "FOUND cloneDirName {0} in {1}", - new Object[]{cloneDirName, targetDir.getPath()}); + LOGGER.log( + Level.FINE, "FOUND cloneDirName {0} in {1}", new Object[] {cloneDirName, targetDir.getPath() + }); } // Strip away the "/./" in "...git-client-plugin/./target/..." return Paths.get(clone.getPath()).normalize().toString(); From a868fa649b2012954a25f6f9d6e0d8eaa10c2afe Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 23 Jul 2026 11:32:57 +0200 Subject: [PATCH 153/155] LegacyCompatibleGitAPIImpl: add support for GITCACHE_MAP_FILENAME to fast-track correct refrepo discovery [JENKINS-64383] The https://github.com/jimklimov/git-refrepo-scripts helper can maintain a file which maps Git URLs to refrepo directory names, and now the plugin can use that file to speed up discovery of the correct dir with less noise in build log when many projects are cached and had to all/many be walked until now. Signed-off-by: Jim Klimov Co-Authored-By: Claude Sonnet 4.6 --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 171 +++++++++++++++++- .../LegacyCompatibleGitAPIImplTest.java | 97 ++++++++++ 2 files changed, 266 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 38383024f3..01afe4a189 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -476,6 +476,15 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * for a needle match or for the big listing. Set to false * when recursing, since this directory was checked already * as part of parent directory inspection. + * + * If a {@value #GITCACHE_MAP_FILENAME} file is present directly under + * referenceBaseDir, it is consulted first for a needle search: a hit there + * (confirmed still a usable git directory which still actually has a + * remote configured for the needle URL, not just a stale filename + * association) is returned immediately, without probing candidate + * subdirectory names or walking the filesystem. A missing file, + * unreadable file, or lack of a (still valid and verified) matching + * entry falls through to the directory-walk search below, unaffected. */ public SimpleEntry>> getSubmodulesUrls( String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { @@ -503,6 +512,20 @@ public SimpleEntry>> getSubmodulesUrls( // return new SimpleEntry<>(false, result); } + // Fast path: if external tooling (e.g. register-git-cache.sh from + // https://github.com/jimklimov/git-refrepo-scripts) maintains a + // GITCACHE_MAP_FILENAME mapping of known remote URLs to their + // subdirectory under this referenceBaseDir, try it before spending + // any I/O on directory probing or `git remote -v` subprocess calls. + if (needle != null && !needle.isEmpty()) { + List mapHit = tryGitCacheMapFile(referenceBaseDirAbs, needle); + if (mapHit != null) { + LinkedHashSet> resultBest = new LinkedHashSet<>(); + resultBest.add(mapHit); + return new SimpleEntry<>(true, resultBest); + } + } + // "this" during a checkout typically represents the job workspace, // but we want to inspect the reference repository located elsewhere // with the same implementation as the end-user set up (CliGit/jGit) @@ -665,8 +688,8 @@ public SimpleEntry>> getSubmodulesUrls( // in or under this directory; however if it is not-null, we still // try to have minimal overhead to complete as soon as we match it. // TODO: Refactor to avoid lookups of dirs that may prove not needed - // in the end (aim for less I/Os to find the goal)... or is one dir - // listing a small price to pay for maintaining one unified logic? + // in the end (aim for less I/Os to find the goal)... or is one dir + // listing a small price to pay for maintaining one unified logic? // If current dir does have submodules, first dig into submodules, // when there is no deeper to drill, report remote URLs and step @@ -685,6 +708,14 @@ public SimpleEntry>> getSubmodulesUrls( // in .gitmodules for a faster possible answer (MODNAME.{url,path} mapping). // See git commit 9cff320a (a couple commits before this line was added) // where a commented-away exploration of how this might be done was removed. + // Note that git does not really support submodule definitions with several + // URLs mapped to same name. Its `git fsck` can complain and "fix" those + // discrepancies, and other operations may look at just some one hit. + + // A different faster-answer idea (external tooling-maintained mapping + // of URL to subdirectory, checked at the top of this method before we + // even get here) is implemented via tryGitCacheMapFile() and does not + // require this repo to have real git submodules at all. // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, @@ -895,6 +926,142 @@ public SimpleEntry>> getSubmodulesUrls( return new SimpleEntry<>(false, result); } + /** Name of an optional mapping file maintained by external tooling (see + * https://github.com/jimklimov/git-refrepo-scripts {@code register-git-cache.sh ls} + * output) directly under a getSubmodulesUrls() referenceBaseDir. + * Each line is tab-separated: remote name, remote URL, and (possibly + * empty, for the top-level referenceBaseDir itself) relative pathname + * of the subdirectory that hosts a git clone with that remote, e.g.: + * 
+     *   repo-1717010012<TAB>ssh://git@example.com/foo/bar.git<TAB>bar.git
+     *
+ * This lets getSubmodulesUrls() resolve a needle URL to its cached + * subdirectory in one file read, instead of probing candidate dirnames + * and running `git remote -v` (a subprocess or JGit call) in each of + * the fanned-out subdirectories. + */ + static final String GITCACHE_MAP_FILENAME = ".gitcache.map"; + + /** Look up {@code needle} in a {@value #GITCACHE_MAP_FILENAME} file + * directly under {@code referenceBaseDirAbs}, if one is present. + * + * @param referenceBaseDirAbs absolute, canonical path to the refrepo + * container directory that might hold the mapping file + * @param needle the (not-null, not-empty) URL we are looking for + * @return a 4-element List as used elsewhere in getSubmodulesUrls() + * (dirname, url, urlNormalized, remoteName) for a hit + * that still resolves to a usable git directory which + * still actually has a remote configured for the needle + * (not just a directory that happens to exist), or + * {@code null} if there is no file, it is unreadable, + * or none of its (still valid) entries match the needle + */ + private List tryGitCacheMapFile(String referenceBaseDirAbs, String needle) { + File mapFile = new File(referenceBaseDirAbs, GITCACHE_MAP_FILENAME); + if (!mapFile.isFile() || !mapFile.canRead()) { + return null; + } + + String needleNorm = normalizeGitUrl(needle, true); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(mapFile), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + String[] cols = line.split("\t", -1); + if (cols.length < 2) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): ignoring malformed " + GITCACHE_MAP_FILENAME + " line: '" + line + + "'"); + continue; + } + String remoteName = cols[0].trim(); + String uri = cols[1].trim(); + String dirRel = cols.length > 2 ? cols[2].trim() : ""; + String uriNorm = normalizeGitUrl(uri, true); + + if (!needleNorm.equals(uriNorm) && !needle.equals(uri)) { + continue; + } + + String dirname = dirRel.isEmpty() ? referenceBaseDirAbs : (referenceBaseDirAbs + "/" + dirRel); + if (getObjectsFile(dirname) == null) { + // Stale entry (dir gone, renamed, or never made) - keep + // looking at remaining lines, then fall back to the walk. + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): " + GITCACHE_MAP_FILENAME + " named '" + dirname + "' (recorded as '" + + remoteName + "') for needle='" + needle + + "' but it is not a usable git dir now; ignoring"); + continue; + } + + // The dir exists and looks like a git repo, but the map file + // could still be stale/misleading (URL changed, remote + // renamed or removed, dir repurposed for another repo since + // the map was last saved) - confirm the needle is actually + // among its configured remotes before trusting this hit. + List verified = verifyGitCacheMapEntry(dirname, needle, needleNorm); + if (verified == null) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): " + GITCACHE_MAP_FILENAME + " named '" + dirname + "' (recorded as '" + + remoteName + "') for needle='" + needle + + "' but it has no matching remote configured now; ignoring"); + continue; + } + + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): resolved needle='" + needle + "' via " + GITCACHE_MAP_FILENAME + + " to dir '" + dirname + "', skipping directory walk"); + return verified; + } + } catch (IOException e) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): failed to read " + GITCACHE_MAP_FILENAME + " at '" + mapFile + "': " + + e.toString()); + } + return null; + } + + /** Confirm that {@code dirname} (already known to be a usable git + * directory) actually has a remote configured whose URL matches + * {@code needle}/{@code needleNorm}, so a {@value #GITCACHE_MAP_FILENAME} + * hit is not trusted purely because the directory happens to still exist. + * + * @return a 4-element List (dirname, url, urlNormalized, remoteName) + * using the remote name actually found in {@code dirname} + * (which may differ from the map file's recorded name, + * e.g. after a manual `git remote rename`), or + * {@code null} if no configured remote there matches + */ + private List verifyGitCacheMapEntry(String dirname, String needle, String needleNorm) { + try { + GitClient g = this.newGit(dirname); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + return List.of(dirname, uri, uriNorm, pair.getValue()); + } + } + } catch (Throwable t) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): failed to verify " + GITCACHE_MAP_FILENAME + " entry in dir '" + dirname + + "' for needle='" + needle + "': " + t.toString()); + } + return null; + } + /** See above. With null needle, returns all data we can find under the * referenceBaseDir tree (can take a while) for the caller to parse */ public SimpleEntry>> getSubmodulesUrls(String referenceBaseDir) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java index c6c37461e1..54b3c278b8 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java @@ -367,6 +367,103 @@ void testFindParameterizedReferenceRepository_submodules_firstEntryUsedForNonExa is(repoFirst.getCanonicalPath())); } + @Test + void testGetSubmodulesUrls_mapFileHit_skipsDirectoryWalk() throws Exception { + String needleUrl = "ssh://git@example.com/org/mapped-project.git"; + File base = newFolder(tempFolder, "refrepo-map-hit"); + File repoMapped = createSubdirWithRemote(base, "mapped-project.git", needleUrl); + // A decoy that would be found first by the ordinary directory walk if + // the mapping file were not consulted, so a wrong hit here would fail + // the assertion on the returned directory below. + createSubdirWithRemote(base, "aaa-decoy", needleUrl); + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-1717010012\t" + needleUrl + "\tmapped-project.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Map file exact match: flag should be true"); + LinkedHashSet> entries = result.getValue(); + assertEquals(1, entries.size(), "Map-file hit should short-circuit with exactly one entry"); + List hit = entries.iterator().next(); + assertThat( + "Map-file hit should point at the mapped dir", + new File(hit.get(0)).getAbsolutePath(), + is(repoMapped.getAbsolutePath())); + assertEquals( + "origin", + hit.get(3), + "remoteName should come from verifying the actual dir's remotes, not the map file's first column"); + } + + @Test + void testGetSubmodulesUrls_mapFileStaleEntry_fallsBackToDirectoryWalk() throws Exception { + String needleUrl = "ssh://git@example.com/org/still-findable.git"; + File base = newFolder(tempFolder, "refrepo-map-stale"); + File repoReal = createSubdirWithRemote(base, "still-findable.git", needleUrl); + // Map file points at a subdirectory that does not actually exist (as if + // the cache had been pruned/renamed after the map was last saved). + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-0000000000\t" + needleUrl + "\tgone-missing.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory-walk fallback should still find the real dir"); + LinkedHashSet> entries = result.getValue(); + assertThat( + "Fallback should find the real dir, not the stale map entry", + new File(entries.iterator().next().get(0)).getAbsolutePath(), + is(repoReal.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_mapFileEntryWithoutMatchingRemote_fallsBackToDirectoryWalk() throws Exception { + // The mapped directory exists and is a real git repo, but its remote + // was changed (or the dir was repurposed) since the map was saved, so + // it no longer actually has the needle URL configured - the map's + // dirname-existence check alone would wrongly trust this hit. + String needleUrl = "ssh://git@example.com/org/moved-away.git"; + File base = newFolder(tempFolder, "refrepo-map-wrong-remote"); + createSubdirWithRemote(base, "moved-away.git", "ssh://git@example.com/org/renamed-elsewhere.git"); + File repoReal = createSubdirWithRemote(base, "aaa-actual-hit", needleUrl); + Files.writeString( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).toPath(), + "repo-1234567890\t" + needleUrl + "\tmoved-away.git\n", + StandardCharsets.UTF_8); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory-walk fallback should still find the real dir"); + LinkedHashSet> entries = result.getValue(); + assertThat( + "Fallback should find the dir that actually has the needle remote, not the mismatched map entry", + new File(entries.iterator().next().get(0)).getAbsolutePath(), + is(repoReal.getAbsolutePath())); + } + + @Test + void testGetSubmodulesUrls_noMapFile_behavesAsBefore() throws Exception { + String needleUrl = "ssh://git@example.com/org/no-map-here.git"; + File base = newFolder(tempFolder, "refrepo-no-map"); + File repoReal = createSubdirWithRemote(base, "no-map-here.git", needleUrl); + assertFalse( + new File(base, LegacyCompatibleGitAPIImpl.GITCACHE_MAP_FILENAME).exists(), + "Precondition: no map file present"); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Directory walk should still find the exact match without a map file"); + assertThat( + new File(result.getValue().iterator().next().get(0)).getAbsolutePath(), is(repoReal.getAbsolutePath())); + } + private static File newFolder(File root, String... subDirs) throws Exception { String subFolder = String.join("/", subDirs); File result = new File(root, subFolder); From be715132a62dc862a50f1b2aaa328e961baf3a89 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 23 Jul 2026 11:57:09 +0200 Subject: [PATCH 154/155] LegacyCompatibleGitAPIImpl: parse repo config file directly to avoid forking git for remote lookups during refrepo walk [JENKINS-64383] getSubmodulesUrls() previously called GitClient.getRemoteUrls() for every candidate directory while walking a fanned-out refrepo, which forks a `git config --local --list` subprocess (CliGit) or opens a full JGit Repository (JGit) per directory. Add getRemoteUrlsFast(), which first tries reading each dir's "config" file directly and parsing it in memory with JGit's own Config/RemoteConfig classes (no subprocess, no Repository object), falling back to the previous GitClient-based lookup when the config is missing, unreadable, unparseable, or uses an include/includeIf directive we do not attempt to resolve here. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Jim Klimov --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 145 +++++++++++++++--- .../LegacyCompatibleGitAPIImplTest.java | 25 +++ 2 files changed, 150 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 01afe4a189..a0dd087943 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -17,9 +17,12 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Paths; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -27,6 +30,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.codec.digest.DigestUtils; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -341,6 +346,116 @@ public static File getObjectsFile(File reference) { return objects; } + /** Get remote name/URL pairs for the git repository at {@code dirname} + * (bare or not, resolved the same way as getObjectsFile()) as fast as + * possible: first try parsing its "config" file directly in memory (no + * subprocess, no JGit Repository object), and only if that is not + * possible or not safe (see {@link #tryReadRemoteUrlsFromConfigFile}), + * fall back to a normal GitClient (CliGit forks `git config --local + * --list`; JGit opens a full Repository) - same as the rest of this + * class did before this fast path was added. + * + *

This is used while walking many candidate directories in + * getSubmodulesUrls(), where most of the cost is exactly this per-dir + * remote lookup. + * + * @return a Map of remote URL to remote name, same shape as + * {@link GitClient#getRemoteUrls()} + */ + private Map getRemoteUrlsFast(String dirname) throws GitException, InterruptedException { + Map uriNames = tryReadRemoteUrlsFromConfigFile(dirname); + if (uriNames != null) { + return uriNames; + } + // "this" during a checkout typically represents the job workspace, + // but here we look at some reference/cache repository elsewhere, + // with the same implementation as the end-user set up (CliGit/jGit) + return this.newGit(dirname).getRemoteUrls(); + } + + /** Try to read remote name/URL pairs directly out of the "config" file + * of the git repository at {@code dirname} (bare or not; resolved to the + * actual GIT_DIR the same way {@link #getObjectsFile} does, so this also + * works for a checked-out submodule's ".git" gitlink file), without + * spawning a git subprocess or opening a full JGit {@link Repository}. + * + * @return a Map of remote URL to remote name (several URL variants per + * remote, mirroring what {@link GitClient#getRemoteUrls()} + * implementations already return, to keep needle matching + * elsewhere in this class equally likely to hit), or + * {@code null} if the config file is missing, unreadable, + * unparseable, or uses {@code include}/{@code includeIf} + * directives that could define remotes elsewhere and + * which we do not attempt to resolve here - callers + * should fall back to a full GitClient-based lookup. + */ + private static Map tryReadRemoteUrlsFromConfigFile(String dirname) { + File objects = getObjectsFile(dirname); + if (objects == null) { + return null; + } + File gitDir = objects.getParentFile(); + if (gitDir == null) { + return null; + } + File configFile = new File(gitDir, "config"); + if (!configFile.isFile() || !configFile.canRead()) { + return null; + } + + String text; + try { + text = new String(Files.readAllBytes(configFile.toPath()), StandardCharsets.UTF_8); + } catch (IOException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to read '" + configFile.getAbsolutePath() + "': " + e.toString()); + return null; + } + + // Bail out rather than risk silently missing remotes actually defined + // in a file pulled in this way - we do not resolve includes here. + if (text.toLowerCase(Locale.ENGLISH).contains("[include")) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): '" + configFile.getAbsolutePath() + + "' has an include/includeIf directive, falling back to GitClient instead of parsing it directly"); + return null; + } + + Config config = new Config(); + try { + config.fromText(text); + } catch (ConfigInvalidException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to parse '" + configFile.getAbsolutePath() + "': " + e.toString()); + return null; + } + + Map uriNames = new LinkedHashMap<>(); + try { + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(config)) { + String remoteName = rc.getName(); + for (URIish u : rc.getURIs()) { + // If uri String values end up identical, Map only stores one entry - + // same trade-off existing GitClient.getRemoteUrls() implementations make. + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException e) { + LOGGER.log( + Level.FINE, + "getRemoteUrlsFast(): failed to interpret remotes from '" + configFile.getAbsolutePath() + "': " + + e.toString()); + return null; + } + return uriNames; + } + /** Handle magic strings in the reference pathname to sort out patterns * classified as evaluated by parametrization, as handled below * @@ -633,19 +748,15 @@ public SimpleEntry>> getSubmodulesUrls( Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing abs refrepo subdir '" + dirname + "'"); - // Use newGit() rather than referenceGit.subGit(dirname) - // since dirname is already an absolute path anyway, and - // it seems that going through subGit() may lose the disk - // letter on Windows agents. - GitClient g = this.newGit(dirname); - LOGGER.log( - Level.FINE, - "getSubmodulesUrls(): checking git workspace in dir '" - + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); + // Prefer parsing the "config" file directly (no subprocess, + // no JGit Repository object) over going through newGit() + // (dirname is already absolute anyway, and going through + // referenceGit.subGit() may lose the disk letter on Windows + // agents) - see getRemoteUrlsFast() for when it falls back. + Map uriNames = getRemoteUrlsFast(dirname); LOGGER.log( Level.FINEST, - "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + "getSubmodulesUrls(): getRemoteUrlsFast() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it @@ -772,15 +883,10 @@ public SimpleEntry>> getSubmodulesUrls( "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); - GitClient g = this.newGit(dirname); - LOGGER.log( - Level.FINE, - "getSubmodulesUrls(): checking git workspace in dir '" - + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); + Map uriNames = getRemoteUrlsFast(dirname); LOGGER.log( Level.FINEST, - "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + "getSubmodulesUrls(): getRemoteUrlsFast() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it @@ -1044,8 +1150,7 @@ private List tryGitCacheMapFile(String referenceBaseDirAbs, String needl */ private List verifyGitCacheMapEntry(String dirname, String needle, String needleNorm) { try { - GitClient g = this.newGit(dirname); - Map uriNames = g.getRemoteUrls(); + Map uriNames = getRemoteUrlsFast(dirname); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java index 54b3c278b8..6f20c76d7d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImplTest.java @@ -464,6 +464,31 @@ void testGetSubmodulesUrls_noMapFile_behavesAsBefore() throws Exception { new File(result.getValue().iterator().next().get(0)).getAbsolutePath(), is(repoReal.getAbsolutePath())); } + @Test + void testGetSubmodulesUrls_configWithIncludeDirective_stillFindsRemoteViaFallback() throws Exception { + // getRemoteUrlsFast() refuses to parse a "config" file with an + // include/includeIf directive directly (remotes could be defined in + // the included file, which it does not resolve), and falls back to + // a full GitClient lookup instead. Confirm that fallback still works + // and still finds the correct match. + String needleUrl = "ssh://git@example.com/org/has-include.git"; + File base = newFolder(tempFolder, "refrepo-config-include"); + File repoDir = createSubdirWithRemote(base, "has-include.git", needleUrl); + File configFile = new File(repoDir, ".git/config"); + Files.writeString( + configFile.toPath(), + "[include]\n\tpath = /nonexistent-included-file-xyz\n", + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND); + + AbstractMap.SimpleEntry>> result = + git.getSubmodulesUrls(base.getAbsolutePath(), needleUrl, true); + + assertTrue(result.getKey(), "Fallback via GitClient should still find the exact match"); + assertThat( + new File(result.getValue().iterator().next().get(0)).getAbsolutePath(), is(repoDir.getAbsolutePath())); + } + private static File newFolder(File root, String... subDirs) throws Exception { String subFolder = String.join("/", subDirs); File result = new File(root, subFolder); From 223927df9104610df28f61e4bc705b5c3cb68567 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 5 Aug 2026 19:18:41 +0200 Subject: [PATCH 155/155] GitClientCloneTest: complete test_clone_reference_parameterized_basename() and fix replaceAll() path corruption it exposed [JENKINS-64383] Code review (PR #644, discussion_r3595155033) flagged that this test only asserted a log message was written and left the real functional checks (alternates file, branches, no stray objects) commented out behind a TODO, so a regression in ${GIT_URL_BASENAME} resolution would still pass. Completing the test surfaced a real bug in LegacyCompatibleGitAPIImpl.findParameterizedReferenceRepository(): the computed basename was passed as the *replacement* argument to String.replaceAll(), but that argument is interpreted as a replacement pattern (backslashes/`$` are escape sequences), not literal text. Any basename containing a backslash - guaranteed for local Windows paths, and possible in principle for any URL - was silently mangled (backslashes stripped), corrupting the resolved reference-repo directory. Fixed all six affected call sites (GIT_URL_BASENAME, its _FALLBACK, and GIT_SUBMODULES/_FALLBACK) by wrapping the replacement with Matcher.quoteReplacement(). The test itself now clones from a forward-slash-spelled local mirror URL (portable on every OS, and avoids the Windows case where a backslash-only path has no '/' to derive a basename from at all), pre-creates the exact refrepo subdirectory production will resolve to - mirroring the existing test_clone_reference_parameterized_sha256() pattern - and re-enables the real assertions instead of the commented-out block. Verified: LegacyCompatibleGitAPIImpl{,JGit,Static}Test (72 tests) and the full GitClientCloneTest suite (69 tests across CliGit/JGit/JGitApache) all pass after `mvn spotless:apply`. Co-Authored-By: Claude Sonnet 5 --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 19 +++-- .../plugins/gitclient/GitClientCloneTest.java | 84 ++++++++++++++----- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index a0dd087943..573dc68889 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Matcher; import org.apache.commons.codec.digest.DigestUtils; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; @@ -1297,9 +1298,11 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", Matcher.quoteReplacement(needleBasename)); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { @@ -1321,9 +1324,11 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", Matcher.quoteReplacement(needleBasename)); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory @@ -1400,9 +1405,11 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll("\\.git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", Matcher.quoteReplacement(needleBasename)); } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES_FALLBACK\\}$", Matcher.quoteReplacement(needleBasename)); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 90125b296e..3bd01ae77a 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -271,36 +271,80 @@ void test_clone_reference() throws Exception { @Test void test_clone_reference_parameterized_basename() throws Exception { + /* This test clones from a local mirror directory rather than a real + * network URL. Git accepts forward slashes in local paths on every OS + * (including Windows), so spell the clone URL with forward slashes: + * this same string is also the "url" argument that + * findParameterizedReferenceRepository() extracts a basename from via + * url.lastIndexOf("/") - a backslash-only Windows path has no '/' to + * split on at all (its "basename" would be the whole path, which is + * not a legal single directory-name component). Forward slashes let + * the basename resolve to a portable value ("clone") on every OS, + * matching how a real repository URL behaves. */ + String wsMirrorUrl = workspace.localMirror().replace(File.separatorChar, '/'); + String wsMirrorBasename = + wsMirrorUrl.substring(wsMirrorUrl.lastIndexOf('/') + 1).replaceAll("\\.git$", ""); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepoBasename.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepoBasename.git/" + wsMirrorBasename); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + LOGGER.log(Level.FINE, "wsRefrepoBase=''{0}''\nwsRefrepo=''{1}''", new Object[] {wsRefrepoBase, wsRefrepo}); + testGitClient .clone_() - .url(workspace.localMirror()) + .url(wsMirrorUrl) .repositoryName("origin") - .reference(workspace.localMirror() + "/${GIT_URL_BASENAME}") + .reference(wsRefrepoBase + "/${GIT_URL_BASENAME}") .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); - check_remote_url(workspace, testGitClient, "origin"); + // Not using check_remote_url(): that helper compares against + // workspace.localMirror()'s OS-separator spelling, but this test + // deliberately clones using the forward-slash spelling above. + assertThat("Remote URL", testGitClient.getRemoteUrl("origin"), is(wsMirrorUrl)); + assertThat("Updated URL", workspace.launchCommand("git", "remote", "-v"), containsString(wsMirrorUrl)); + // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); + + LOGGER.log(Level.FINE, "clone output:\n======\n{0}\n======", messages); + assertThat( - "Reference repo name-parsing logged in: " + messages, + "Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")), handler.containsMessageSubstring("Parameterized reference path ") - && handler.containsMessageSubstring(" replaced with: "), + && handler.containsMessageSubstring(" replaced with: ") + && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)), is(true)); - // TODO: Actually construct the local filesystem path to match - // the last pathname component from the URL (plus/minus ".git" - // extension). Be sure to clean away this path at end of test, - // so that the test_clone_reference_parameterized_basename_fallback() - // below is not confused - it expects this location to not exist. - // Skip: Missing if clone failed - currently would, with bogus - // path above and not yet pre-created path structure. - /* - assertThat("Reference repo logged in: " + messages, - handler.containsMessageSubstring( - "Using reference repository: "), is(true)); - assertAlternateFilePointsToLocalMirror(); - assertBranchesExist(testGitClient.getBranches(), "master"); - assertNoObjectsInRepository(); - */ + + if (wsRefrepo != null) { + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure } @Test