Skip to content

Commit

Permalink
feat: add BASE_IMAGEs to image cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@KyleAure
KyleAure committed Jan 22, 2025
1 parent 5c700f0 commit 674b32b
Show file tree
Hide file tree
Showing 8 changed files with 235 additions and 183 deletions.
17 changes: 15 additions & 2 deletions dev/fattest.simplicity/src/componenttest/containers/ImageBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ private static String constructResource(DockerImageName image) {
* @return The substituted docker image of the BASE_IMAGE argument
*/
private static DockerImageName findBaseImageFrom(String resource) {
final String BASE_IMAGE_PREFIX = "ARG BASE_IMAGE=\"";

/*
* Finds the Dockerfile on classpath and will extract the file to a temporary location so we can read it.
* This will be done during the image build step anyway so this is just front-loading that work for our benefit.
Expand All @@ -185,10 +187,12 @@ private static DockerImageName findBaseImageFrom(String resource) {
String errorMessage = "The Dockerfile did not contain a BASE_IMAGE argument declaration. "
+ "This is required to allow us to pull and substitute the BASE_IMAGE using the ImageNameSubstitutor.";

String baseImageName = dockerfileLines.filter(line -> line.startsWith("ARG BASE_IMAGE"))
String baseImageLine = dockerfileLines.filter(line -> line.startsWith("ARG BASE_IMAGE"))
.findFirst()
.orElseThrow(() -> new IllegalStateException(errorMessage));

String baseImageName = baseImageLine.substring(BASE_IMAGE_PREFIX.length(), baseImageLine.lastIndexOf('"'));

return ImageNameSubstitutor.instance().apply(DockerImageName.parse(baseImageName));
}

Expand Down Expand Up @@ -227,6 +231,15 @@ protected String getDescription() {
return "ImageBuilderSubstitutor with registry " + REGISTRY;
}

}
// Hide instance method from parent class.
// which will choose the ImageNameSubstitutor based on environment.
private static ImageBuilderSubstitutor instance;

public static synchronized ImageNameSubstitutor instance() {
if (Objects.isNull(instance)) {
instance = new ImageBuilderSubstitutor();
}
return instance;
}
}
}
2 changes: 1 addition & 1 deletion dev/fattest.simplicity/test/componenttest/containers/ImageBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public void imageBuilderSubstitutorErrorTest() {
}

private DockerImageName getImage(ImageBuilder builder) throws Exception {
Field image = builder.getClass().getField("image");
Field image = builder.getClass().getDeclaredField("image");
image.setAccessible(true);
return (DockerImageName) image.get(builder);
}
Expand Down
13 changes: 12 additions & 1 deletion dev/io.openliberty.org.testcontainers/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,17 @@ task assembleTestContainerData(type: JavaExec) {
classpath = configurations.runtime.plus(sourceSets.main.runtimeClasspath)
main = "io.openliberty.org.testcontainers.generate.CacheFiles"
args project.getProjectDir()

// Debugging properties for testcontainers
systemProperties = [
'org.slf4j.simpleLogger.logFile': new File(project.getProjectDir(), 'build/CacheFiles.log'),
'org.slf4j.simpleLogger.showDateTime': 'true',
'org.slf4j.simpleLogger.dateTimeFormat': '[MM/dd/yyyy HH:mm:ss:SSS z]',
'org.slf4j.simpleLogger.log.org.testcontainers': 'debug',
'org.slf4j.simpleLogger.log.tc': 'debug',
'org.slf4j.simpleLogger.log.com.github.dockerjava': 'warn',
'org.slf4j.simpleLogger.log.com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire': 'off'
]
}

task generateCustomImages(type: JavaExec) {
Expand All @@ -51,7 +62,7 @@ task generateCustomImages(type: JavaExec) {

// Debugging properties for testcontainers
systemProperties = [
'org.slf4j.simpleLogger.logFile': new File(project.getProjectDir(), 'build/testcontainers.log'),
'org.slf4j.simpleLogger.logFile': new File(project.getProjectDir(), 'build/CustomImages.log'),
'org.slf4j.simpleLogger.showDateTime': 'true',
'org.slf4j.simpleLogger.dateTimeFormat': '[MM/dd/yyyy HH:mm:ss:SSS z]',
'org.slf4j.simpleLogger.log.org.testcontainers': 'debug',
Expand Down
5 changes: 5 additions & 0 deletions dev/io.openliberty.org.testcontainers/cache/externals
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
# NOTICE: This file was automatically updated to reflect changes made to test projects.
# Please check these changes into GitHub
confluentinc/cp-kafka:7.1.1
docker.io/ibmcom/cloudant-developer:2.0.1
elastic/logstash:7.16.3
fbicodex/spnego-kdc-server:1.0
ghcr.io/gvenzl/oracle-free:23.5-full-faststart
ghcr.io/gvenzl/oracle-free:23.5-slim-faststart
icr.io/db2_community/db2:11.5.9.0
icr.io/db2_community/db2:12.1.0.0
infinispan/server:10.0.1.Final
infinispan/server:13.0.10.Final
jaegertracing/all-in-one:1.54
Expand Down Expand Up @@ -33,8 +35,11 @@ openzipkin/zipkin-slim:2.23
otel/opentelemetry-collector-contrib:0.103.0
otel/opentelemetry-collector:0.74.0
public.ecr.aws/docker/library/alpine:3.17
public.ecr.aws/docker/library/couchdb:3.2.0
public.ecr.aws/docker/library/mongo:6.0.6
public.ecr.aws/docker/library/postgres:17.0
public.ecr.aws/docker/library/postgres:17.0-alpine
public.ecr.aws/docker/library/python:3.11.6
ryanesch/acme-boulder:1.2
seleniarm/standalone-chromium:4.8.3
selenium/standalone-chrome:4.8.3
Expand Down
5 changes: 5 additions & 0 deletions dev/io.openliberty.org.testcontainers/cache/images
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
# NOTICE: This file was automatically updated to reflect changes made to test projects.
# Please check these changes into GitHub
wasliberty-aws-docker-remote/docker/library/alpine:3.17
wasliberty-aws-docker-remote/docker/library/couchdb:3.2.0
wasliberty-aws-docker-remote/docker/library/mongo:6.0.6
wasliberty-aws-docker-remote/docker/library/postgres:17.0
wasliberty-aws-docker-remote/docker/library/postgres:17.0-alpine
wasliberty-aws-docker-remote/docker/library/python:3.11.6
wasliberty-docker-remote/ibmcom/cloudant-developer:2.0.1
wasliberty-ghcr-docker-remote/gvenzl/oracle-free:23.5-full-faststart
wasliberty-ghcr-docker-remote/gvenzl/oracle-free:23.5-slim-faststart
wasliberty-icr-docker-remote/db2_community/db2:11.5.9.0
wasliberty-icr-docker-remote/db2_community/db2:12.1.0.0
wasliberty-infrastructure-docker/confluentinc/cp-kafka:7.1.1
wasliberty-infrastructure-docker/elastic/logstash:7.16.3
wasliberty-infrastructure-docker/fbicodex/spnego-kdc-server:1.0
Expand Down
10 changes: 9 additions & 1 deletion ...liberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CacheFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import java.io.File;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -109,7 +111,13 @@ public static void main(String[] args) {
}
}

//TODO Add all BASE_IMAGE arguments from Dockerfiles in this project
// Investigate all Dockerfiles and add the BASE_NAME to externals list
Path commonPath = Paths.get(projectPath, "resources", "openliberty", "testcontainers");
Dockerfile.findDockerfiles(commonPath).stream()
.map(location -> new Dockerfile(location))
.forEach(dockerfile -> {
externals.add(dockerfile.baseImageName.asCanonicalNameString());
});

String header = "# NOTICE: This file was automatically updated to reflect changes made to test projects." + System.lineSeparator() +
"# Please check these changes into GitHub" + System.lineSeparator();
Expand Down
180 changes: 2 additions & 178 deletions ...berty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CustomImages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,10 @@
*******************************************************************************/
package io.openliberty.org.testcontainers.generate;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;

import org.testcontainers.images.PullPolicy;
import org.testcontainers.images.builder.ImageFromDockerfile;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.ImageNameSubstitutor;

/**
* Allows external contributors a convenient way to build the custom images we
Expand Down Expand Up @@ -52,7 +43,7 @@ public static void main(String[] args) {
Path commonPath = Paths.get(projectPath, "resources", "openliberty", "testcontainers");

// Construct a list of Dockerfiles
findDockerfiles(commonPath).stream()
Dockerfile.findDockerfiles(commonPath).stream()
.map(location -> new Dockerfile(location))
.forEach(dockerfile -> {
// Find or build all images
Expand All @@ -64,7 +55,7 @@ public static void main(String[] args) {

ImageFromDockerfile img = new ImageFromDockerfile(dockerfile.imageName.asCanonicalNameString(), false)
.withDockerfile(dockerfile.location)
.withBuildArg(BASE_IMAGE, dockerfile.baseName.asCanonicalNameString());
.withBuildArg(BASE_IMAGE, dockerfile.baseImageNameSubstituted.asCanonicalNameString());

try {
System.out.println("Building image: " + dockerfile.imageName.asCanonicalNameString());
Expand All @@ -80,171 +71,4 @@ public static void main(String[] args) {
long end = System.currentTimeMillis();
System.out.println("Execution time in ms: " + (end - start));
}

/**
* Walk through all files nested within a shared path and find every Dockerfile.
*
* @param commonPath the shared path within which all Dockerfiles are nested
* within
* @return A list of paths to every Dockerfile
*/
private static List<Path> findDockerfiles(Path commonPath) {
final String FILE_NAME = "Dockerfile";
final List<Path> Dockerfiles = new ArrayList<>();

try (Stream<Path> paths = Files.walk(commonPath)) {
paths.filter(Files::isRegularFile).filter(path -> path.getFileName().toString().endsWith(FILE_NAME))
.forEach(Dockerfiles::add);
} catch (IOException e) {
throw new RuntimeException("Error searching files: " + e.getMessage());
}

return Dockerfiles;
}

/**
* A record of a Dockerfile
*/
private static class Dockerfile {
public final Path location;
public final DockerImageName imageName;
public final DockerImageName baseName;

public Dockerfile(Path location) {
this.location = location;
this.imageName = constructImageName(location);
this.baseName = findBaseImageFrom(location);
}

public boolean isCached() {
if (PullPolicy.defaultPolicy().shouldPull(imageName)) {
System.out.println("Did not find image locally: " + imageName.asCanonicalNameString());
return false;
} else {
System.out.println("Found image locally: " + imageName.asCanonicalNameString());
return true;
}
}

/**
* Using the Dockerfile's path, parse the directory structure to construct a
* fully qualified DockerImageName to be associated with this Dockerfile.
*
* @param location of Dockerfile to be built
* @return The DockerImageName for this Dockerfile
*/
private static DockerImageName constructImageName(Path location) {

// io.openliberty.org.testcontainers/resources/openliberty/testcontainers/[repository]/[version]/Dockerfile
final String fullPath = location.toString();

// Find version (between the last two backslash / characters)
int end = fullPath.lastIndexOf('/');
int start = fullPath.substring(0, end).lastIndexOf('/') + 1;
final String version = fullPath.substring(start, end);

// Find repository (between "resources/" and version)
start = fullPath.lastIndexOf("resources/") + 10;
end = fullPath.indexOf(version) - 1;
final String repository = fullPath.substring(start, end);

// Construct and return name
DockerImageName name = ImageBuilderSubstitutor.instance()
.apply(DockerImageName.parse(repository).withTag(version));
System.out.println("DockerImageName from path: " + name.asCanonicalNameString());
return name;
}

/**
* Similar logic to ImageBuilder.findBaseImageFrom(resource)
*
* However, in this case we can only use the ArtifactoryMirrorSubstitutor so we
* have to manually put in the Artifactory registry (when available)
*
* @param location of Dockerfile the resource path of the Dockerfile
* @return The substituted docker image of the BASE_IMAGE argument
*/
private DockerImageName findBaseImageFrom(Path location) {

final String BASE_IMAGE_PREFIX = "ARG BASE_IMAGE=\"";
final String ARTIFACTORY_REGISTRY = System.getenv("ARTIFACTORY_REGISTRY");

Stream<String> dockerfileLines;
try {
dockerfileLines = Files.readAllLines(location).stream();
} catch (IOException e) {
throw new RuntimeException("Could not read or find Dockerfile in " + location.toString(), e);
}

String errorMessage = "The Dockerfile did not contain a BASE_IMAGE argument declaration. "
+ "This is required to allow us to pull and substitute the BASE_IMAGE using the ImageNameSubstitutor.";

String baseImageLine = dockerfileLines.filter(line -> line.startsWith(BASE_IMAGE_PREFIX)).findFirst()
.orElseThrow(() -> new IllegalStateException(errorMessage));

String baseImageName = baseImageLine.substring(BASE_IMAGE_PREFIX.length(), baseImageLine.lastIndexOf('"'));

DockerImageName original = DockerImageName.parse(baseImageName);
DockerImageName substituted = ImageNameSubstitutor.instance().apply(original);

if (original.equals(substituted)) {
System.out.println("Keep original BASE_IMAGE: " + original.asCanonicalNameString());
return original;
} else {
// Substitutor was used, also prepend the registry.
System.out.println("Use substituted BASE_IMAGE: " + substituted.asCanonicalNameString());
return substituted.withRegistry(ARTIFACTORY_REGISTRY);
}
}
}

/**
* A ImageNameSubstitutor for images built by this outer class.
*/
private static class ImageBuilderSubstitutor extends ImageNameSubstitutor {

// TODO replace with the finalized property expected on our build systems
private static final String INTERNAL_REGISTRY_ENV = "INTERNAL_REGISTRY";

// Ensures when we look for cached images Docker only attempt to find images
// locally or from an internally configured registry.
private static final String REGISTRY = System.getenv(INTERNAL_REGISTRY_ENV) == null
? "localhost"
: System.getenv(INTERNAL_REGISTRY_ENV);

// The repository where all Open Liberty images will be cached
private static final String REPOSITORY_PREFIX = "openliberty/testcontainers/";

@Override
public DockerImageName apply(final DockerImageName original) {
Objects.requireNonNull(original);

if (!original.getRegistry().isEmpty()) {
throw new IllegalArgumentException("DockerImageName with the registry " + original.getRegistry()
+ " cannot be substituted with registry " + REGISTRY);
}

if (original.getRepository().startsWith(REPOSITORY_PREFIX)) {
return original.withRegistry(REGISTRY);
} else {
return original.withRepository(REPOSITORY_PREFIX + original.getRepository()).withRegistry(REGISTRY);
}
}

@Override
protected String getDescription() {
return "ImageBuilderSubstitutor with registry " + REGISTRY;
}

// Hide instance method from parent class.
// which will choose the ImageNameSubstitutor based on environment.
private static ImageBuilderSubstitutor instance;

public static synchronized ImageNameSubstitutor instance() {
if (Objects.isNull(instance)) {
instance = new ImageBuilderSubstitutor();
}
return instance;
}
}
}
Loading

0 comments on commit 674b32b

Please sign in to comment.