diff --git a/srcdeps-core/src/main/java/org/srcdeps/core/impl/scm/JGitScm.java b/srcdeps-core/src/main/java/org/srcdeps/core/impl/scm/JGitScm.java index 0a7b9fa..c283745 100644 --- a/srcdeps-core/src/main/java/org/srcdeps/core/impl/scm/JGitScm.java +++ b/srcdeps-core/src/main/java/org/srcdeps/core/impl/scm/JGitScm.java @@ -17,18 +17,23 @@ package org.srcdeps.core.impl.scm; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; import java.util.Collection; +import java.util.List; import java.util.Set; import javax.inject.Named; import javax.inject.Singleton; -import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.FetchCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; @@ -56,26 +61,15 @@ @Singleton public class JGitScm implements Scm { private static final Logger log = LoggerFactory.getLogger(JGitScm.class); + private static final String REMOTE = "remote"; private static final String SCM_GIT_PREFIX = "git:"; - public static String getScmGitPrefix() { - return SCM_GIT_PREFIX; - } - private static final String SRCDEPS_WORKING_BRANCH = "srcdeps-working-branch"; - /** - * @return srcdeps will use this branch to perform its magic - */ - public static String getSrcdepsWorkingBranch() { - return SRCDEPS_WORKING_BRANCH; - } - /** * Tells if the given filesystem directory contains a valid git repository. * - * @param dir - * a {@link Path} to check + * @param dir a {@link Path} to check * @return {@code true} if the given {@code dir} contains a valid git repository; {@code false} otherwise. */ private static boolean containsGitRepo(Path dir) { @@ -96,21 +90,62 @@ private static boolean containsGitRepo(Path dir) { } } + static void ensureRemoteAvailable(String useUrl, String remoteAlias, Git git) throws IOException { + final StoredConfig config = git.getRepository().getConfig(); + boolean save = false; + final String foundUrl = config.getString(REMOTE, remoteAlias, "url"); + if (!useUrl.equals(foundUrl)) { + config.setString(REMOTE, remoteAlias, "url", useUrl); + save = true; + } + final String foundFetch = config.getString(REMOTE, remoteAlias, "fetch"); + final String expectedFetch = "+refs/heads/*:refs/remotes/" + remoteAlias + "/*"; + if (!expectedFetch.equals(foundFetch)) { + config.setString(REMOTE, remoteAlias, "fetch", expectedFetch); + save = true; + } + if (save) { + config.save(); + } + } + + public static String getScmGitPrefix() { + return SCM_GIT_PREFIX; + } + + /** + * @return srcdeps will use this branch to perform its magic + */ + public static String getSrcdepsWorkingBranch() { + return SRCDEPS_WORKING_BRANCH; + } + private static String stripUriPrefix(String url) { return url.substring(SCM_GIT_PREFIX.length()); } + /** + * @param url the git URL to generate a remote alias for + * @return a Byte64 encoded sha1 hash of the given {@code url} prefixed with {@code origin-} + */ + static String toRemoteAlias(String url) { + try { + final MessageDigest sha1Digest = MessageDigest.getInstance("SHA-1"); + sha1Digest.update(url.getBytes(StandardCharsets.UTF_8)); + final byte[] bytes = sha1Digest.digest(); + return "origin-" + Base64.getUrlEncoder().encodeToString(bytes); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + /** * Makes sure that the given {@code refToFind} is available in the {@code advertisedRefs}. * - * @param advertisedRefs - * the {@link Collection} of {@link Ref}s to search in - * @param refToFind - * the ref name to find - * @param url - * the URL used to fetch - * @throws ScmException - * if the given {@code refToFind} could not be found in the {@code advertisedRefs} + * @param advertisedRefs the {@link Collection} of {@link Ref}s to search in + * @param refToFind the ref name to find + * @param url the URL used to fetch + * @throws ScmException if the given {@code refToFind} could not be found in the {@code advertisedRefs} */ private void assertRefFetched(Collection advertisedRefs, String refToFind, String url) throws ScmException { for (Ref ref : advertisedRefs) { @@ -124,17 +159,12 @@ private void assertRefFetched(Collection advertisedRefs, String refToFind, /** * Walks back through the history of the {@code advertisedRefs} and tries to find the given {@code commitSha1}. * - * @param repository - * the current {@link Repository} to search in - * @param advertisedRefs - * the list of refs that were fetched and whose histories should be searched through - * @param commitSha1 - * the commit to find - * @param url - * the URL that was used to fetch - * @throws ScmException - * if the given {@code commitSha1} could not be found in the history of any of the - * {@code advertisedRefs} + * @param repository the current {@link Repository} to search in + * @param advertisedRefs the list of refs that were fetched and whose histories should be searched through + * @param commitSha1 the commit to find + * @param url the URL that was used to fetch + * @throws ScmException if the given {@code commitSha1} could not be found in the history of any of the + * {@code advertisedRefs} */ private void assertRevisionFetched(Repository repository, Collection advertisedRefs, String commitSha1, String url) throws ScmException { @@ -165,78 +195,45 @@ private void assertRevisionFetched(Repository repository, Collection advert * {@link BuildRequest#getSrcVersion()} of the given {@code request}. *

* This implementation first checks if {@code request.getProjectRootDirectory()} returns a directory containing a - * valid git repository. If it does, git fetch operation is used. Otherwise, the repository is cloned. + * valid git repository. If it does not, git init operation is invoked. After that git fetch and git reset are used + * to checkout the sources. * - * @param request - * determines the project to checkout + * @param request determines the project to checkout * @return the {@code commitId} the {@code HEAD} points at - * @throws ScmException - * on any SCM related problem + * @throws ScmException on any SCM related problem * @see org.srcdeps.core.Scm#checkout(org.srcdeps.core.BuildRequest) */ @Override public String checkout(BuildRequest request) throws ScmException { - - Path dir = request.getProjectRootDirectory(); - boolean dirExists = Files.exists(dir); - final String result; - if (dirExists && containsGitRepo(dir)) { - /* there is a valid repo - try to fetch and reset */ - result = fetchAndReset(request); - } else { + final Path dir = request.getProjectRootDirectory(); + int i = 0; + final List urls = request.getScmUrls(); + if (!Files.exists(dir) || !containsGitRepo(dir)) { /* there is no valid git repo in the directory */ try { SrcdepsCoreUtils.ensureDirectoryExistsAndEmpty(dir); } catch (IOException e) { - throw new ScmException(String.format("srcdeps could not create directory [%s]", dir), e); + throw new ScmException(String.format("Could not create directory [%s]", dir), e); } - result = cloneAndCheckout(request); - } - return result; - } - - String cloneAndCheckout(BuildRequest request) throws ScmException { - final Path dir = request.getProjectRootDirectory(); - - final SrcVersion srcVersion = request.getSrcVersion(); - ScmException lastException = null; - - /* Try the urls one after another and exit on the first success */ - for (String url : request.getScmUrls()) { - String useUrl = stripUriPrefix(url); - log.info("srcdeps: Attempting to clone ref [{}] from SCM URL [{}]", request.getSrcVersion(), useUrl); - - CloneCommand cmd = Git.cloneRepository().setURI(useUrl).setDirectory(dir.toFile()); - - final WellKnownType t = srcVersion.getWellKnownType(); - switch (t) { - case branch: - case tag: - cmd.setBranch(srcVersion.getScmVersion()); - break; - case revision: - cmd.setCloneAllBranches(true); - break; - default: - throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" + t + "'."); + try (Git git = Git.init().setDirectory(dir.toFile()).call()) { + } catch (GitAPIException e) { + throw new ScmException(String.format("Could not init a git repository in [%s]", dir), e); } - - try (Git git = cmd.call()) { - git.checkout().setName(srcVersion.getScmVersion()).call(); - /* return on the first success */ - final Ref ref = git.getRepository().exactRef("HEAD"); - return ref.getObjectId().getName(); - } catch (Exception e) { - log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(), - useUrl, e.getClass().getName(), e.getMessage()); - lastException = new ScmException(String.format("Could not checkout from URL [%s]", useUrl), e); + } + for (String url : urls) { + final String useUrl = stripUriPrefix(url); + final String result = fetchAndReset(useUrl, i, urls.size(), request.getSrcVersion(), dir); + if (result != null) { + return result; } + i++; } - throw lastException; + throw new ScmException( + String.format("Could not checkout [%s] from URLs %s", request.getSrcVersion(), request.getScmUrls())); } - String fetchAndReset(BuildRequest request) throws ScmException { - final Path dir = request.getProjectRootDirectory(); + String fetchAndReset(String useUrl, int urlIndex, int urlCount, SrcVersion srcVersion, Path dir) + throws ScmException { /* Forget local changes */ try (Git git = Git.open(dir.toFile())) { Set removedFiles = git.clean().setCleanDirectories(true).call(); @@ -253,82 +250,76 @@ String fetchAndReset(BuildRequest request) throws ScmException { log.warn(String.format("srcdeps: Could not forget local changes in [%s]", dir), e); } - final SrcVersion srcVersion = request.getSrcVersion(); - ScmException lastException = null; - int i = 0; - for (String url : request.getScmUrls()) { - String useUrl = stripUriPrefix(url); - log.info("srcdeps: Attempting to fetch version [{}] from SCM URL [{}]", request.getSrcVersion(), useUrl); - String remoteAlias = i == 0 ? "origin" : "origin" + i; - try (Git git = Git.open(dir.toFile())) { + log.info("srcdeps: Fetching version [{}] from SCM URL {}/{} [{}]", srcVersion, urlIndex + 1, urlCount, useUrl); + final String remoteAlias = toRemoteAlias(useUrl); + try (Git git = Git.open(dir.toFile())) { - StoredConfig config = git.getRepository().getConfig(); - config.setString("remote", remoteAlias, "url", useUrl); - config.save(); + ensureRemoteAvailable(useUrl, remoteAlias, git); - final String scmVersion = srcVersion.getScmVersion(); - final String startPoint; - final String refToFetch; - FetchCommand fetch = git.fetch().setRemote(remoteAlias); - switch (srcVersion.getWellKnownType()) { - case branch: - refToFetch = "refs/heads/" + scmVersion; - fetch.setRefSpecs(new RefSpec("+refs/heads/" + scmVersion + ":refs/remotes/" + remoteAlias + "/" - + scmVersion)); - startPoint = remoteAlias + "/" + scmVersion; - break; - case tag: - refToFetch = "refs/tags/" + scmVersion; - fetch.setRefSpecs(new RefSpec(refToFetch)); - startPoint = scmVersion; - break; - case revision: - refToFetch = null; - startPoint = scmVersion; - break; - default: - throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" - + srcVersion.getWellKnownType() + "'."); - } - FetchResult fetchResult = fetch.call(); - - /* - * Let's check that the desired startPoint was really fetched from the current URL. Otherwise, the - * startPoint may come from an older fetch of the same repo URL (but was removed in between) or it may - * come from an older fetch of another URL. These cases may introduce situations when one developer can - * see a successful srcdep build (because he still has the outdated ref in his local git repo) but - * another dev with exectly the same setup cannot checkout because the ref is not there in any of the - * remote repos anymore. - */ - Collection advertisedRefs = fetchResult.getAdvertisedRefs(); - switch (srcVersion.getWellKnownType()) { - case branch: - case tag: - assertRefFetched(advertisedRefs, refToFetch, url); - break; - case revision: - assertRevisionFetched(git.getRepository(), advertisedRefs, scmVersion, url); - break; - default: - throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" - + srcVersion.getWellKnownType() + "'."); - } + final String scmVersion = srcVersion.getScmVersion(); + final String startPoint; + final String refToFetch; + final FetchCommand fetch = git.fetch().setRemote(remoteAlias); + switch (srcVersion.getWellKnownType()) { + case branch: + refToFetch = "refs/heads/" + scmVersion; + fetch.setRefSpecs( + new RefSpec("+refs/heads/" + scmVersion + ":refs/remotes/" + remoteAlias + "/" + scmVersion)); + startPoint = remoteAlias + "/" + scmVersion; + break; + case tag: + refToFetch = "refs/tags/" + scmVersion; + fetch.setRefSpecs(new RefSpec(refToFetch)); + startPoint = scmVersion; + break; + case revision: + refToFetch = null; + startPoint = scmVersion; + break; + default: + throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" + + srcVersion.getWellKnownType() + "'."); + } + FetchResult fetchResult = fetch.call(); + + /* + * Let's check that the desired startPoint was really fetched from the current URL. Otherwise, the + * startPoint may come from an older fetch of the same repo URL (but was removed in between) or it may come + * from an older fetch of another URL. These cases may introduce situations when one developer can see a + * successful srcdep build (because he still has the outdated ref in his local git repo) but another dev + * with exectly the same setup cannot checkout because the ref is not there in any of the remote repos + * anymore. + */ + Collection advertisedRefs = fetchResult.getAdvertisedRefs(); + switch (srcVersion.getWellKnownType()) { + case branch: + case tag: + assertRefFetched(advertisedRefs, refToFetch, useUrl); + break; + case revision: + assertRevisionFetched(git.getRepository(), advertisedRefs, scmVersion, useUrl); + break; + default: + throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" + + srcVersion.getWellKnownType() + "'."); + } - git.reset().setMode(ResetType.HARD).setRef(startPoint).call(); - final Ref ref = git.getRepository().exactRef("HEAD"); - return ref.getObjectId().getName(); - } catch (ScmException e) { - log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(), - useUrl, e.getClass().getName(), e.getMessage()); - lastException = e; - } catch (Exception e) { - log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(), - useUrl, e.getClass().getName(), e.getMessage()); - lastException = new ScmException(String.format("Could not checkout from URL [%s]", useUrl), e); + git.reset().setMode(ResetType.HARD).setRef(startPoint).call(); + final Ref ref = git.getRepository().exactRef("HEAD"); + return ref.getObjectId().getName(); + } catch (ScmException e) { + final String msg = String.format("srcdeps: Could not checkout [%s] from SCM URL %d/%d [%s]", srcVersion, + urlIndex + 1, urlCount, useUrl); + if (urlIndex + 1 == urlCount) { + throw new ScmException(msg, e); + } else { + log.warn(msg, e); } - i++; + } catch (Exception e) { + throw new ScmException(String.format("Could not checkout [%s] from SCM URL %d/%d [%s]", srcVersion, + urlIndex + 1, urlCount, useUrl), e); } - throw lastException; + return null; } @Override diff --git a/srcdeps-core/src/test/java/org/srcdeps/core/impl/scm/JGitScmTest.java b/srcdeps-core/src/test/java/org/srcdeps/core/impl/scm/JGitScmTest.java index 1b6cce5..b5658be 100644 --- a/srcdeps-core/src/test/java/org/srcdeps/core/impl/scm/JGitScmTest.java +++ b/srcdeps-core/src/test/java/org/srcdeps/core/impl/scm/JGitScmTest.java @@ -51,7 +51,6 @@ private void assertCommit(Path dir, String expectedSha1) throws IOException, NoH Assert.assertEquals(String.format("Git repository in [%s] not at the expected revision", dir), expectedSha1, foundSha1); } - } @Test @@ -100,7 +99,7 @@ public void testCheckout() throws IOException, ScmException, NoHeadException, Gi .gradleModelTransformer(CharStreamSource.defaultModelTransformer()) // .build(); - commitId = jGitScm.fetchAndReset(fetchingRequest); + commitId = jGitScm.checkout(fetchingRequest); Assert.assertEquals("0a5ab902099b24c2b13ed1dad8c5f537458bcc89", commitId); /* ensure that the WC's HEAD has the known commit hash */ @@ -115,7 +114,7 @@ public void testCheckout() throws IOException, ScmException, NoHeadException, Gi .gradleModelTransformer(CharStreamSource.defaultModelTransformer()) // .build(); - commitId = jGitScm.fetchAndReset(fetchBranchRequest); + commitId = jGitScm.checkout(fetchBranchRequest); Assert.assertEquals("a84403b6fb44c5a588a9fe39d939c977e1e5c6a4", commitId); /* ensure that the WC's HEAD has the known commit hash */ @@ -131,9 +130,7 @@ public void testCheckout() throws IOException, ScmException, NoHeadException, Gi expectedCommit = git.commit().setMessage("Added test.txt").call().getId().getName(); } - System.out.println("expectedCommit = "+ expectedCommit); - - commitId = jGitScm.fetchAndReset(fetchBranchRequest); + commitId = jGitScm.checkout(fetchBranchRequest); Assert.assertEquals(expectedCommit, commitId); /* Reset back the morning-branch */ @@ -141,9 +138,91 @@ public void testCheckout() throws IOException, ScmException, NoHeadException, Gi git.reset().setMode(ResetType.HARD).setRef("a84403b6fb44c5a588a9fe39d939c977e1e5c6a4").call(); } - commitId = jGitScm.fetchAndReset(fetchBranchRequest); + commitId = jGitScm.checkout(fetchBranchRequest); Assert.assertEquals("a84403b6fb44c5a588a9fe39d939c977e1e5c6a4", commitId); assertCommit(dir, "a84403b6fb44c5a588a9fe39d939c977e1e5c6a4"); } + + @Test + public void testCheckoutMultiUrl() throws IOException, ScmException, NoHeadException, GitAPIException { + + /* Create a local clone */ + final Path localGitRepos = targetDir.resolve("local-git-repos"); + final Path srcdepsTestArtifactDirectory0 = localGitRepos.resolve("srcdeps-test-artifact-testCheckoutMultiUrl-0"); + SrcdepsCoreUtils.deleteDirectory(srcdepsTestArtifactDirectory0); + final Path srcdepsTestArtifactDirectory1 = localGitRepos.resolve("srcdeps-test-artifact-testCheckoutMultiUrl-1"); + SrcdepsCoreUtils.deleteDirectory(srcdepsTestArtifactDirectory1); + + final String remoteGitUri = "https://github.com/srcdeps/srcdeps-test-artifact.git"; + final String mornigBranch = "morning-branch"; + try (Git git = Git.cloneRepository().setURI(remoteGitUri).setDirectory(srcdepsTestArtifactDirectory0.toFile()) + .setCloneAllBranches(true).call()) { + git.checkout().setName(mornigBranch).setCreateBranch(true).setStartPoint("origin/" + mornigBranch).call(); + } + SrcdepsCoreUtils.copyDirectory(srcdepsTestArtifactDirectory0, srcdepsTestArtifactDirectory1); + + final String localGitUri0 = srcdepsTestArtifactDirectory0.resolve(".git").toUri().toString(); + final String localGitUri1 = srcdepsTestArtifactDirectory1.resolve(".git").toUri().toString(); + + /* Add a commit to repo 1 */ + final Path testTxtPath = srcdepsTestArtifactDirectory1.resolve("test.txt"); + Files.write(testTxtPath, "Test0".getBytes(StandardCharsets.UTF_8)); + + final Path dir = targetDir.resolve("test-repo-testCheckoutMultiUrl"); + SrcdepsCoreUtils.ensureDirectoryExistsAndEmpty(dir); + { + final String addedCommitId; + try (Git git = Git.init().setDirectory(srcdepsTestArtifactDirectory1.toFile()).call()) { + git.add().addFilepattern(testTxtPath.getFileName().toString()).call(); + addedCommitId = git.commit().setMessage("Added test.txt").call().getId().getName(); + } + + /* first clone */ + final BuildRequest cloningRequest = BuildRequest.builder() // + .srcVersion(SrcVersion.parse("0.0.1-SRC-revision-"+ addedCommitId)) // + .dependentProjectRootDirectory(dir) // + .projectRootDirectory(dir) // + .scmUrl("git:" + localGitUri0) // + .scmUrl("git:" + localGitUri1) // + .versionsMavenPluginVersion(Maven.getDefaultVersionsMavenPluginVersion()) // + .gradleModelTransformer(CharStreamSource.defaultModelTransformer()) // + .build(); + final JGitScm jGitScm = new JGitScm(); + + final String commitId = jGitScm.checkout(cloningRequest); + Assert.assertEquals(addedCommitId, commitId); + } + + /* Add one more commit to repo 1 */ + Files.write(testTxtPath, "Test1".getBytes(StandardCharsets.UTF_8)); + { + final String addedCommitId; + try (Git git = Git.init().setDirectory(srcdepsTestArtifactDirectory1.toFile()).call()) { + git.add().addFilepattern(testTxtPath.getFileName().toString()).call(); + addedCommitId = git.commit().setMessage("Changed test.txt").call().getId().getName(); + } + + /* fetch and reset */ + final BuildRequest fetchResetRequest = BuildRequest.builder() // + .srcVersion(SrcVersion.parse("0.0.1-SRC-revision-"+ addedCommitId)) // + .dependentProjectRootDirectory(dir) // + .projectRootDirectory(dir) // + .scmUrl("git:" + localGitUri0) // + .scmUrl("git:" + localGitUri1) // + .versionsMavenPluginVersion(Maven.getDefaultVersionsMavenPluginVersion()) // + .gradleModelTransformer(CharStreamSource.defaultModelTransformer()) // + .build(); + final JGitScm jGitScm = new JGitScm(); + + final String commitId = jGitScm.checkout(fetchResetRequest); + Assert.assertEquals(addedCommitId, commitId); + } + + } + + @Test + public void toRemoteAlias() { + Assert.assertEquals("origin-OracyX45LTLgEE14zEKVWpi-CTg=", JGitScm.toRemoteAlias("https://github.com/srcdeps/srcdeps-test-artifact.git")); + } }