Skip to content
This repository has been archived by the owner on Sep 20, 2023. It is now read-only.

Commit

Permalink
Merge pull request #37 from ppalaga/i36
Browse files Browse the repository at this point in the history 
Fix #36 Handle multiple SCM URLs properly
  • Loading branch information
@ppalaga
ppalaga authored Aug 19, 2018
2 parents 157c977 + dc9ee07 commit ecbdd91
Show file tree
Hide file tree
Showing 2 changed files with 237 additions and 167 deletions.
311 changes: 151 additions & 160 deletions srcdeps-core/src/main/java/org/srcdeps/core/impl/scm/JGitScm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,23 @@
package org.srcdeps.core.impl.scm;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.Collection;
import java.util.List;
import java.util.Set;

import javax.inject.Named;
import javax.inject.Singleton;

import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.FetchCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ResetCommand.ResetType;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
Expand Down Expand Up @@ -56,26 +61,15 @@
@Singleton
public class JGitScm implements Scm {
private static final Logger log = LoggerFactory.getLogger(JGitScm.class);
private static final String REMOTE = "remote";
private static final String SCM_GIT_PREFIX = "git:";

public static String getScmGitPrefix() {
return SCM_GIT_PREFIX;
}

private static final String SRCDEPS_WORKING_BRANCH = "srcdeps-working-branch";

/**
* @return srcdeps will use this branch to perform its magic
*/
public static String getSrcdepsWorkingBranch() {
return SRCDEPS_WORKING_BRANCH;
}

/**
* Tells if the given filesystem directory contains a valid git repository.
*
* @param dir
* a {@link Path} to check
* @param dir a {@link Path} to check
* @return {@code true} if the given {@code dir} contains a valid git repository; {@code false} otherwise.
*/
private static boolean containsGitRepo(Path dir) {
Expand All @@ -96,21 +90,62 @@ private static boolean containsGitRepo(Path dir) {
}
}

static void ensureRemoteAvailable(String useUrl, String remoteAlias, Git git) throws IOException {
final StoredConfig config = git.getRepository().getConfig();
boolean save = false;
final String foundUrl = config.getString(REMOTE, remoteAlias, "url");
if (!useUrl.equals(foundUrl)) {
config.setString(REMOTE, remoteAlias, "url", useUrl);
save = true;
}
final String foundFetch = config.getString(REMOTE, remoteAlias, "fetch");
final String expectedFetch = "+refs/heads/*:refs/remotes/" + remoteAlias + "/*";
if (!expectedFetch.equals(foundFetch)) {
config.setString(REMOTE, remoteAlias, "fetch", expectedFetch);
save = true;
}
if (save) {
config.save();
}
}

public static String getScmGitPrefix() {
return SCM_GIT_PREFIX;
}

/**
* @return srcdeps will use this branch to perform its magic
*/
public static String getSrcdepsWorkingBranch() {
return SRCDEPS_WORKING_BRANCH;
}

private static String stripUriPrefix(String url) {
return url.substring(SCM_GIT_PREFIX.length());
}

/**
* @param url the git URL to generate a remote alias for
* @return a Byte64 encoded sha1 hash of the given {@code url} prefixed with {@code origin-}
*/
static String toRemoteAlias(String url) {
try {
final MessageDigest sha1Digest = MessageDigest.getInstance("SHA-1");
sha1Digest.update(url.getBytes(StandardCharsets.UTF_8));
final byte[] bytes = sha1Digest.digest();
return "origin-" + Base64.getUrlEncoder().encodeToString(bytes);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}

/**
* Makes sure that the given {@code refToFind} is available in the {@code advertisedRefs}.
*
* @param advertisedRefs
* the {@link Collection} of {@link Ref}s to search in
* @param refToFind
* the ref name to find
* @param url
* the URL used to fetch
* @throws ScmException
* if the given {@code refToFind} could not be found in the {@code advertisedRefs}
* @param advertisedRefs the {@link Collection} of {@link Ref}s to search in
* @param refToFind the ref name to find
* @param url the URL used to fetch
* @throws ScmException if the given {@code refToFind} could not be found in the {@code advertisedRefs}
*/
private void assertRefFetched(Collection<Ref> advertisedRefs, String refToFind, String url) throws ScmException {
for (Ref ref : advertisedRefs) {
Expand All @@ -124,17 +159,12 @@ private void assertRefFetched(Collection<Ref> advertisedRefs, String refToFind,
/**
* Walks back through the history of the {@code advertisedRefs} and tries to find the given {@code commitSha1}.
*
* @param repository
* the current {@link Repository} to search in
* @param advertisedRefs
* the list of refs that were fetched and whose histories should be searched through
* @param commitSha1
* the commit to find
* @param url
* the URL that was used to fetch
* @throws ScmException
* if the given {@code commitSha1} could not be found in the history of any of the
* {@code advertisedRefs}
* @param repository the current {@link Repository} to search in
* @param advertisedRefs the list of refs that were fetched and whose histories should be searched through
* @param commitSha1 the commit to find
* @param url the URL that was used to fetch
* @throws ScmException if the given {@code commitSha1} could not be found in the history of any of the
* {@code advertisedRefs}
*/
private void assertRevisionFetched(Repository repository, Collection<Ref> advertisedRefs, String commitSha1,
String url) throws ScmException {
Expand Down Expand Up @@ -165,78 +195,45 @@ private void assertRevisionFetched(Repository repository, Collection<Ref> advert
* {@link BuildRequest#getSrcVersion()} of the given {@code request}.
* <p>
* This implementation first checks if {@code request.getProjectRootDirectory()} returns a directory containing a
* valid git repository. If it does, git fetch operation is used. Otherwise, the repository is cloned.
* valid git repository. If it does not, git init operation is invoked. After that git fetch and git reset are used
* to checkout the sources.
*
* @param request
* determines the project to checkout
* @param request determines the project to checkout
* @return the {@code commitId} the {@code HEAD} points at
* @throws ScmException
* on any SCM related problem
* @throws ScmException on any SCM related problem
* @see org.srcdeps.core.Scm#checkout(org.srcdeps.core.BuildRequest)
*/
@Override
public String checkout(BuildRequest request) throws ScmException {

Path dir = request.getProjectRootDirectory();
boolean dirExists = Files.exists(dir);
final String result;
if (dirExists && containsGitRepo(dir)) {
/* there is a valid repo - try to fetch and reset */
result = fetchAndReset(request);
} else {
final Path dir = request.getProjectRootDirectory();
int i = 0;
final List<String> urls = request.getScmUrls();
if (!Files.exists(dir) || !containsGitRepo(dir)) {
/* there is no valid git repo in the directory */
try {
SrcdepsCoreUtils.ensureDirectoryExistsAndEmpty(dir);
} catch (IOException e) {
throw new ScmException(String.format("srcdeps could not create directory [%s]", dir), e);
throw new ScmException(String.format("Could not create directory [%s]", dir), e);
}
result = cloneAndCheckout(request);
}
return result;
}

String cloneAndCheckout(BuildRequest request) throws ScmException {
final Path dir = request.getProjectRootDirectory();

final SrcVersion srcVersion = request.getSrcVersion();
ScmException lastException = null;

/* Try the urls one after another and exit on the first success */
for (String url : request.getScmUrls()) {
String useUrl = stripUriPrefix(url);
log.info("srcdeps: Attempting to clone ref [{}] from SCM URL [{}]", request.getSrcVersion(), useUrl);

CloneCommand cmd = Git.cloneRepository().setURI(useUrl).setDirectory(dir.toFile());

final WellKnownType t = srcVersion.getWellKnownType();
switch (t) {
case branch:
case tag:
cmd.setBranch(srcVersion.getScmVersion());
break;
case revision:
cmd.setCloneAllBranches(true);
break;
default:
throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '" + t + "'.");
try (Git git = Git.init().setDirectory(dir.toFile()).call()) {
} catch (GitAPIException e) {
throw new ScmException(String.format("Could not init a git repository in [%s]", dir), e);
}

try (Git git = cmd.call()) {
git.checkout().setName(srcVersion.getScmVersion()).call();
/* return on the first success */
final Ref ref = git.getRepository().exactRef("HEAD");
return ref.getObjectId().getName();
} catch (Exception e) {
log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(),
useUrl, e.getClass().getName(), e.getMessage());
lastException = new ScmException(String.format("Could not checkout from URL [%s]", useUrl), e);
}
for (String url : urls) {
final String useUrl = stripUriPrefix(url);
final String result = fetchAndReset(useUrl, i, urls.size(), request.getSrcVersion(), dir);
if (result != null) {
return result;
}
i++;
}
throw lastException;
throw new ScmException(
String.format("Could not checkout [%s] from URLs %s", request.getSrcVersion(), request.getScmUrls()));
}

String fetchAndReset(BuildRequest request) throws ScmException {
final Path dir = request.getProjectRootDirectory();
String fetchAndReset(String useUrl, int urlIndex, int urlCount, SrcVersion srcVersion, Path dir)
throws ScmException {
/* Forget local changes */
try (Git git = Git.open(dir.toFile())) {
Set<String> removedFiles = git.clean().setCleanDirectories(true).call();
Expand All @@ -253,82 +250,76 @@ String fetchAndReset(BuildRequest request) throws ScmException {
log.warn(String.format("srcdeps: Could not forget local changes in [%s]", dir), e);
}

final SrcVersion srcVersion = request.getSrcVersion();
ScmException lastException = null;
int i = 0;
for (String url : request.getScmUrls()) {
String useUrl = stripUriPrefix(url);
log.info("srcdeps: Attempting to fetch version [{}] from SCM URL [{}]", request.getSrcVersion(), useUrl);
String remoteAlias = i == 0 ? "origin" : "origin" + i;
try (Git git = Git.open(dir.toFile())) {
log.info("srcdeps: Fetching version [{}] from SCM URL {}/{} [{}]", srcVersion, urlIndex + 1, urlCount, useUrl);
final String remoteAlias = toRemoteAlias(useUrl);
try (Git git = Git.open(dir.toFile())) {

StoredConfig config = git.getRepository().getConfig();
config.setString("remote", remoteAlias, "url", useUrl);
config.save();
ensureRemoteAvailable(useUrl, remoteAlias, git);

final String scmVersion = srcVersion.getScmVersion();
final String startPoint;
final String refToFetch;
FetchCommand fetch = git.fetch().setRemote(remoteAlias);
switch (srcVersion.getWellKnownType()) {
case branch:
refToFetch = "refs/heads/" + scmVersion;
fetch.setRefSpecs(new RefSpec("+refs/heads/" + scmVersion + ":refs/remotes/" + remoteAlias + "/"
+ scmVersion));
startPoint = remoteAlias + "/" + scmVersion;
break;
case tag:
refToFetch = "refs/tags/" + scmVersion;
fetch.setRefSpecs(new RefSpec(refToFetch));
startPoint = scmVersion;
break;
case revision:
refToFetch = null;
startPoint = scmVersion;
break;
default:
throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '"
+ srcVersion.getWellKnownType() + "'.");
}
FetchResult fetchResult = fetch.call();

/*
* Let's check that the desired startPoint was really fetched from the current URL. Otherwise, the
* startPoint may come from an older fetch of the same repo URL (but was removed in between) or it may
* come from an older fetch of another URL. These cases may introduce situations when one developer can
* see a successful srcdep build (because he still has the outdated ref in his local git repo) but
* another dev with exectly the same setup cannot checkout because the ref is not there in any of the
* remote repos anymore.
*/
Collection<Ref> advertisedRefs = fetchResult.getAdvertisedRefs();
switch (srcVersion.getWellKnownType()) {
case branch:
case tag:
assertRefFetched(advertisedRefs, refToFetch, url);
break;
case revision:
assertRevisionFetched(git.getRepository(), advertisedRefs, scmVersion, url);
break;
default:
throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '"
+ srcVersion.getWellKnownType() + "'.");
}
final String scmVersion = srcVersion.getScmVersion();
final String startPoint;
final String refToFetch;
final FetchCommand fetch = git.fetch().setRemote(remoteAlias);
switch (srcVersion.getWellKnownType()) {
case branch:
refToFetch = "refs/heads/" + scmVersion;
fetch.setRefSpecs(
new RefSpec("+refs/heads/" + scmVersion + ":refs/remotes/" + remoteAlias + "/" + scmVersion));
startPoint = remoteAlias + "/" + scmVersion;
break;
case tag:
refToFetch = "refs/tags/" + scmVersion;
fetch.setRefSpecs(new RefSpec(refToFetch));
startPoint = scmVersion;
break;
case revision:
refToFetch = null;
startPoint = scmVersion;
break;
default:
throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '"
+ srcVersion.getWellKnownType() + "'.");
}
FetchResult fetchResult = fetch.call();

/*
* Let's check that the desired startPoint was really fetched from the current URL. Otherwise, the
* startPoint may come from an older fetch of the same repo URL (but was removed in between) or it may come
* from an older fetch of another URL. These cases may introduce situations when one developer can see a
* successful srcdep build (because he still has the outdated ref in his local git repo) but another dev
* with exectly the same setup cannot checkout because the ref is not there in any of the remote repos
* anymore.
*/
Collection<Ref> advertisedRefs = fetchResult.getAdvertisedRefs();
switch (srcVersion.getWellKnownType()) {
case branch:
case tag:
assertRefFetched(advertisedRefs, refToFetch, useUrl);
break;
case revision:
assertRevisionFetched(git.getRepository(), advertisedRefs, scmVersion, useUrl);
break;
default:
throw new IllegalStateException("Unexpected " + WellKnownType.class.getName() + " value '"
+ srcVersion.getWellKnownType() + "'.");
}

git.reset().setMode(ResetType.HARD).setRef(startPoint).call();
final Ref ref = git.getRepository().exactRef("HEAD");
return ref.getObjectId().getName();
} catch (ScmException e) {
log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(),
useUrl, e.getClass().getName(), e.getMessage());
lastException = e;
} catch (Exception e) {
log.warn("srcdeps: Could not checkout version [{}] from SCM URL [{}]: [{}]: [{}]", request.getSrcVersion(),
useUrl, e.getClass().getName(), e.getMessage());
lastException = new ScmException(String.format("Could not checkout from URL [%s]", useUrl), e);
git.reset().setMode(ResetType.HARD).setRef(startPoint).call();
final Ref ref = git.getRepository().exactRef("HEAD");
return ref.getObjectId().getName();
} catch (ScmException e) {
final String msg = String.format("srcdeps: Could not checkout [%s] from SCM URL %d/%d [%s]", srcVersion,
urlIndex + 1, urlCount, useUrl);
if (urlIndex + 1 == urlCount) {
throw new ScmException(msg, e);
} else {
log.warn(msg, e);
}
i++;
} catch (Exception e) {
throw new ScmException(String.format("Could not checkout [%s] from SCM URL %d/%d [%s]", srcVersion,
urlIndex + 1, urlCount, useUrl), e);
}
throw lastException;
return null;
}

@Override
Expand Down
Loading

0 comments on commit ecbdd91

Please sign in to comment.