From c59ad4650abcd98481d4cf0aed5f76bb24495f31 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Fri, 24 Nov 2023 18:33:10 -0500 Subject: [PATCH 01/12] * Started working on new instrumenting plugin and driver --- .github/workflows/main.yml | 3 +- .../cs/psl/phosphor/Instrumenter.java | 123 ++------- .../columbia/cs/psl/phosphor/OptionsUtil.java | 101 +++++++ integration-tests/README.md | 4 + integration-tests/pom.xml | 216 ++++----------- phosphor-instrument-jigsaw/pom.xml | 92 +++++-- .../phosphor/instrumenter/JLinkInvoker.java | 43 +-- .../phosphor/instrumenter/PhosphorPacker.java | 3 +- .../StandaloneJVMInstrumenter.java | 35 --- .../phosphor/jlink}/DeletingFileVisitor.java | 5 +- .../swe/phosphor/jlink/InstrumentDriver.java | 54 ++++ .../phosphor/jlink/InstrumentJLinkPlugin.java | 128 +++++++++ .../swe/phosphor/jlink/InstrumentUtil.java | 87 ++++++ .../swe/phosphor/jlink/Instrumentation.java | 30 ++ .../gmu/swe/phosphor/jlink/Instrumenter.java | 186 +++++++++++++ .../gmu/swe/phosphor/jlink/JLinkInvoker.java | 68 +++++ .../jlink/JLinkRegistrationAgent.java | 247 +++++++++++++++++ .../jlink/PhosphorInstrumentation.java | 35 +++ .../src/main/java/module-info.java | 22 +- phosphor-instrument-maven-plugin/pom.xml | 99 ++----- ...ExecutePermissionAssigningFileVisitor.java | 28 -- .../ignored/maven/PhosphorInstrumentUtil.java | 184 ------------- .../maven/PhosphorInstrumentingMojo.java | 258 ------------------ .../instrument/InstrumentingMojo.java | 221 +++++++++++++++ pom.xml | 36 ++- 25 files changed, 1370 insertions(+), 938 deletions(-) create mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java create mode 100644 integration-tests/README.md delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/StandaloneJVMInstrumenter.java rename {phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven => phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink}/DeletingFileVisitor.java (94%) create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java delete mode 100644 phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/ExecutePermissionAssigningFileVisitor.java delete mode 100644 phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentUtil.java delete mode 100644 phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentingMojo.java create mode 100644 phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49a321e45..8dad8ce55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,6 @@ jobs: strategy: fail-fast: false matrix: -# java: [ '8', '11', '16', '17' ] java: [ '8', '11', '16' ] steps: - uses: actions/checkout@v3 @@ -22,7 +21,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Run tests - run: mvn install -ntp -Ddacapo.skip=false + run: mvn install -ntp -Pdacapo deploy: runs-on: self-hosted needs: build-and-test diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java index 71f023c72..d513f81db 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java @@ -11,7 +11,6 @@ import org.objectweb.asm.tree.MethodNode; import java.io.*; -import java.lang.instrument.ClassFileTransformer; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; @@ -26,14 +25,6 @@ import static edu.columbia.cs.psl.phosphor.Configuration.taintTagFactoryPackage; public class Instrumenter { - // jmod magic number and version number, taken from java.base/jdk/internal/jmod/JmodFile.java - private static final int JMOD_MAJOR_VERSION = 0x01; - private static final int JMOD_MINOR_VERSION = 0x00; - public static final byte[] JMOD_MAGIC_NUMBER = { - 0x4A, 0x4D, /* JM */ - JMOD_MAJOR_VERSION, JMOD_MINOR_VERSION, /* version 1.0 */ - }; - public static ClassLoader loader; public static Map classes = Collections.synchronizedMap(new HashMap<>()); public static InputStream sourcesFile; @@ -41,7 +32,6 @@ public class Instrumenter { public static InputStream taintThroughFile; static String curPath; static int n = 0; - private static ClassFileTransformer addlTransformer; static { classes.putAll(ClassSupertypeReadingTransformer.classNodes); @@ -52,37 +42,6 @@ private Instrumenter() { // Prevents this class from being instantiated } - public static void preAnalysis() { - - } - - public static void finishedAnalysis() { - System.out.println("Analysis Completed: Beginning Instrumentation Phase"); - } - - public static boolean isCollection(String internalName) { - try { - Class c; - if(TaintTrackingClassVisitor.IS_RUNTIME_INST && !internalName.startsWith("java/")) { - return false; - } - c = Class.forName(internalName.replace("/", "."), false, loader); - if(java.util.Collection.class.isAssignableFrom(c)) { - return true; - } - } catch(Throwable ex) { - // - } - return false; - } - - public static boolean isClassWithHashMapTag(String clazz) { - return clazz.startsWith("java/lang/Boolean") - || clazz.startsWith("java/lang/Character") - || clazz.startsWith("java/lang/Byte") - || clazz.startsWith("java/lang/Short"); - } - public static boolean isIgnoredClass(String owner) { return Configuration.taintTagFactory.isIgnoredClass(owner) || taintTagFactoryPackage != null && StringUtils.startsWith(owner, taintTagFactoryPackage) @@ -90,12 +49,10 @@ public static boolean isIgnoredClass(String owner) { || (Configuration.controlFlowManager != null && Configuration.controlFlowManager.isIgnoredClass(owner)) || (Configuration.ADDL_IGNORE != null && StringUtils.startsWith(owner, Configuration.ADDL_IGNORE)) //|| !owner.startsWith("edu/columbia/cs/psl") - /* - For these classes: HotSpot expects fields to be at hardcoded offsets of these classes. If we instrument - them, it will break those assumptions and segfault. - - Different classes have different assumptions, and there are special cases for these elsewhere in Phosphor. - */ + // For these classes: HotSpot expects fields to be at hardcoded offsets of these classes. + // If we instrument them, it will break those assumptions and segfault. + // Different classes have different assumptions, and there are special cases for these elsewhere in + // Phosphor. || StringUtils.startsWith(owner, "java/lang/Object") || StringUtils.startsWith(owner, "java/lang/Boolean") || StringUtils.startsWith(owner, "java/lang/Character") @@ -105,22 +62,23 @@ public static boolean isIgnoredClass(String owner) { || StringUtils.startsWith(owner, "java/lang/ref/Reference") || StringUtils.startsWith(owner, "java/lang/ref/FinalReference") || StringUtils.startsWith(owner, "java/lang/ref/SoftReference") - || StringUtils.equals(owner, "java/lang/invoke/LambdaForm") //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool - || StringUtils.startsWith(owner, "java/lang/invoke/LambdaForm$") //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool - /* - Phosphor internal methods - */ + //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool + || StringUtils.equals(owner, "java/lang/invoke/LambdaForm") + //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool + || StringUtils.startsWith(owner, "java/lang/invoke/LambdaForm$") + // Phosphor internal classes || StringUtils.startsWith(owner, "edu/columbia/cs/psl/phosphor") || StringUtils.startsWith(owner, "edu/gmu/swe/phosphor/ignored") - /* - Reflection is handled by: - DelagatingMethod/ConstructorAccessorImpl calls either NativeMethod/ConstructorAccessorImpl OR - calls GeneratedMethod/ConstructorAccessorImpl. We do the wrapping at the delagating level. - Generated code won't be instrumented. Hence, it's convenient to also not wrap the native version, - so that passed params/returns line up exactly when we hit the reflected call - */ + // Reflection is handled by: + // DelegatingMethod/ConstructorAccessorImpl calls either NativeMethod/ConstructorAccessorImpl OR + // calls GeneratedMethod/ConstructorAccessorImpl. + // We do the wrapping at the delegating level. + // Generated code won't be instrumented. + // Hence, it's convenient to also not wrap the native version, + // so that passed params/returns line up exactly when we hit the reflected call || StringUtils.startsWith(owner, "edu/columbia/cs/psl/phosphor/struct/TaintedWith") - || StringUtils.startsWith(owner, "jdk/internal/misc/UnsafeConstants"); //Java 9+ class full of hardcoded offsets + // Java 9+ class full of hardcoded offsets + || StringUtils.startsWith(owner, "jdk/internal/misc/UnsafeConstants"); } public static byte[] instrumentClass(String path, InputStream is, boolean renameInterfaces) { @@ -141,19 +99,12 @@ public static byte[] instrumentClass(String path, InputStream is, boolean rename buffer.flush(); PreMain.PCLoggingTransformer transformer = new PreMain.PCLoggingTransformer(); byte[] ret = transformer.transform(Instrumenter.loader, path, null, null, buffer.toByteArray(), false); - if(addlTransformer != null) { - byte[] ret2 = addlTransformer.transform(Instrumenter.loader, path, null, null, ret); - if(ret2 != null) { - ret = ret2; - } - } curPath = null; return ret; - } catch(Exception ex) { + } catch(Exception e) { curPath = null; - ex.printStackTrace(); - //return null; - throw new IllegalStateException(); + e.printStackTrace(); + throw new IllegalStateException(e); } } @@ -237,37 +188,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { } catch(MalformedURLException e1) { e1.printStackTrace(); } - if(args.length == 3) { - System.out.println("Using extra classpath file: " + args[2]); - try { - File f = new File(args[2]); - if(f.exists() && f.isFile()) { - for(Scanner s = new Scanner(f); s.hasNextLine();) { - urls.add(new File(s.nextLine()).getCanonicalFile().toURI().toURL()); - } - } else if(f.isDirectory()) { - urls.add(f.toURI().toURL()); - } - } catch(IOException e) { - e.printStackTrace(); - } - } else if(args.length > 3) { - for(int i = 2; i < args.length; i++) { - File f = new File(args[i]); - if(!f.exists()) { - System.err.println("Unable to read path " + args[i]); - System.exit(-1); - } - if(f.isDirectory() && !f.getAbsolutePath().endsWith("/")) { - f = new File(f.getAbsolutePath() + "/"); - } - try { - urls.add(f.getCanonicalFile().toURI().toURL()); - } catch(Exception ex) { - ex.printStackTrace(); - } - } - } + URL[] urlArray = new URL[urls.size()]; urlArray = urls.toArray(urlArray); diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java new file mode 100644 index 000000000..4818d1a26 --- /dev/null +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java @@ -0,0 +1,101 @@ +package edu.columbia.cs.psl.phosphor; + +import edu.columbia.cs.psl.phosphor.struct.SinglyLinkedList; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.File; +import java.util.*; + +public class OptionsUtil { + /** + * Creates a standardized copy of the specified properties where each Phosphor option without an argument is mapped + * to either true or not present in the properties, each Phosphor option with an argument is either mapped to a + * non-null, non-empty string or not present in properties, and no other keys are present in the properties. + * + * @param isRuntimeInst true if the options should be standardized against the options available during dynamic + * instrumentation, otherwise standardizes against the options available during static + * instrumentation + * @param properties un-standardized properties to be standardized + * @return a standardized copy of properties + */ + public static Properties canonicalizeProperties(Properties properties, boolean isRuntimeInst) { + Set propNames = properties.stringPropertyNames(); + Properties canonicalProps = new Properties(); + Map phosphorOptionMap = createPhosphorOptionMap(isRuntimeInst); + for (String propName : propNames) { + if (phosphorOptionMap.containsKey(propName)) { + Option option = phosphorOptionMap.get(propName); + if (option.hasArg()) { + if (properties.getProperty(propName) != null + && !properties.getProperty(propName).isEmpty()) { + canonicalProps.setProperty(option.getOpt(), properties.getProperty(propName)); + } + } else { + if (properties.getProperty(propName).isEmpty() + || "true".equalsIgnoreCase(properties.getProperty(propName))) { + canonicalProps.setProperty(option.getOpt(), "true"); + } + } + } else { + System.err.println("Unknown Phosphor option: " + propName); + } + } + return canonicalProps; + } + + /** + * @param isRuntimeInst true if a map of options available during dynamic instrumentation should be returned, + * otherwise a map of option available during static instrumentation should be returned + * @return a mapping from the names of configuration options available in Phosphor to an instance of + * org.apache.commons.cli.Option that represents that configuration option + */ + private static Map createPhosphorOptionMap(boolean isRuntimeInst) { + Map phosphorOptionMap = new HashMap<>(); + Options options = PhosphorOption.createOptions(isRuntimeInst); + for (Option option : options.getOptions()) { + phosphorOptionMap.put(option.getOpt(), option); + if (option.hasLongOpt()) { + phosphorOptionMap.put(option.getLongOpt(), option); + } + } + return phosphorOptionMap; + } + + public static String[] createPhosphorMainArguments( + File inputDirectory, File outputDirectory, Properties properties) { + properties = canonicalizeProperties(properties, false); + SinglyLinkedList arguments = new SinglyLinkedList<>(); + Set propNames = properties.stringPropertyNames(); + for (String propName : propNames) { + arguments.addLast("-" + propName); + if (!"true".equals(properties.getProperty(propName))) { + arguments.addLast(properties.getProperty(propName)); + } + } + arguments.addLast(inputDirectory.getAbsolutePath()); + arguments.addLast(outputDirectory.getAbsolutePath()); + return arguments.toArray(new String[0]); + } + + public static Set> getConfigurationClasses(Properties properties) { + Set> classes = new HashSet<>(); + Options options = PhosphorOption.createOptions(false); + for (Option option : options.getOptions()) { + if (option.getType().equals(Class.class)) { + String key = option.getOpt(); + if (properties.containsKey(key)) { + try { + Class clazz = Class.forName(properties.getProperty(key)); + classes.add(clazz); + } catch (ReflectiveOperationException e) { + String message = + String.format("Failed to create %s class: %s", key, properties.getProperty(key)); + throw new IllegalArgumentException(message, e); + } + } + } + } + return classes; + } +} diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 000000000..9deace86e --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,4 @@ +## Tips + +To skip data flow instrumentation and tests add the option `-P-data-flow` to the Maven commands. +This will disable the `data-flow` profile. \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index a4aeb8048..d2faebcfb 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -10,21 +10,39 @@ 0.1.0-SNAPSHOT - true - false - false - false + ${project.build.testSourceDirectory}/../resources - taintSources=${basedir}/src/test/resources/taint-sources,taintSinks=${basedir}/src/test/resources/taint-sinks,taintThrough=${basedir}/src/test/resources/taint-through + taintSources=${test.resources.dir}/taint-sources,taintSinks=${test.resources.dir}/taint-sinks,taintThrough=${test.resources.dir}/taint-through - ${project.build.directory}/cached-data-flows - ${project.build.directory}/cached-instrumentations - ${project.build.directory}/cached-control-flows - jvm-inst-data - jvm-inst-control + ${project.build.directory}/phosphor/data/jdk/ + ${project.build.directory}/phosphor/data/cache + false + + + edu.gmu.swe.phosphor + Phosphor + ${project.version} + + + edu.gmu.swe.phosphor + phosphor-jigsaw-javaagent + ${project.version} + + + + maven-dependency-plugin + + + process-resources + + properties + + + + org.apache.maven.plugins maven-compiler-plugin @@ -43,47 +61,25 @@ ${project.version} - data-flows + data-flow instrument none - ${data.flow.jvm} + ${data.flow.jdk} + + ${data.flow.cache} + - - - - + true + true + true + true ALL-MODULE-PATH - - - ${data.flow.cache} - - false - - - - - - - - - - - - - - - - - - - - @@ -96,33 +92,8 @@ true - ${phosphor.jar} + ${edu.gmu.swe.phosphor:Phosphor:jar} - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.8 - - - process-resources - - run - - - true - - - - - - - - - - java9Plus @@ -131,52 +102,19 @@ false - ${phosphor-javaagent.jar} + ${edu.gmu.swe.phosphor:phosphor-jigsaw-javaagent:jar} - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.8 - - - process-resources - - run - - - true - - - - - - - - - - - - run-dacapo - - - dacapo.skip - false - - + dacapo edu.gmu.swe.phosphor phosphor-instrument-maven-plugin - ${project.version} - data-flows + data-flow process-test-resources @@ -192,13 +130,12 @@ exec - ${dacapo.skip} ${basedir}/runDacapo.sh ${project.build.directory} - ${phosphor.jar} - ${phosphor.javaagent} - ${project.build.directory}/${data.flow.jvm}/bin/java + ${edu.gmu.swe.phosphor:Phosphor:jar} + ${phosphor.agent} + ${data.flow.jdk}/bin/java ${isJava8} @@ -209,7 +146,7 @@ - run-tests + data-flow !skipTests @@ -220,14 +157,9 @@ edu.gmu.swe.phosphor phosphor-instrument-maven-plugin - ${project.version} - data-flows - process-test-resources - - - control-flows + data-flow process-test-resources @@ -237,34 +169,18 @@ maven-failsafe-plugin - integration-test-data-flows + data-flow - ${skipDataFlowTests} - ${project.build.directory}/${data.flow.jvm}/bin/java + ${data.flow.jdk}/bin/java **/*ObjTagITCase.java - - false - - ${argLine.prefix},enum,acmpeq,cacheDir=${data.flow.cache} - - - - integration-test - verify - - - - integration-test-instrumentation - - ${skipInstrumentationTest} - ${project.build.directory}/${data.flow.jvm}/bin/java - **/*InstCase.java false - ${argLine.prefix},enum,acmpeq,cacheDir=${instrumentation.cache} + -DphosphorCacheDirectory=${data.flow.cache} + -Xbootclasspath/a:${edu.gmu.swe.phosphor:Phosphor:jar} + -javaagent:${phosphor.agent}=${auto.taint},enum,acmpeq @@ -272,40 +188,10 @@ verify - - - - - - - - - - - - - - - - - - - - - edu.gmu.swe.phosphor - Phosphor - ${project.version} - - - edu.gmu.swe.phosphor - phosphor-jigsaw-javaagent - ${project.version} - - diff --git a/phosphor-instrument-jigsaw/pom.xml b/phosphor-instrument-jigsaw/pom.xml index 3169721cc..3f62873ba 100644 --- a/phosphor-instrument-jigsaw/pom.xml +++ b/phosphor-instrument-jigsaw/pom.xml @@ -2,42 +2,69 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 phosphor-instrument-jigsaw - jar - A JLink Plugin for Creating Phosphor-instrumented JVMs + Tools for assisting in the creation of instrumented JVMs edu.gmu.swe.phosphor phosphor-parent 0.1.0-SNAPSHOT + + + edu.gmu.swe.phosphor + Phosphor + ${project.version} + + + org.jacoco + org.jacoco.core + 0.8.11 + + + org.apache.maven.plugins maven-compiler-plugin - 3.8.0 - 9 - 9 - - - --add-exports=jdk.jlink/jdk.tools.jlink.plugin=edu.columbia.cs.psl.jigsaw.phosphor.instrumenter - - - --add-reads=edu.columbia.cs.psl.jigsaw.phosphor.instrumenter=ALL-UNNAMED - - + module-info.java + 8 + 8 true + + org.apache.maven.plugins + maven-enforcer-plugin + + + default-cli + validate + + enforce + + + + + [9,) + + + + + + org.apache.maven.plugins maven-jar-plugin - edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.JLinkRegistrationAgent - edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.StandaloneJVMInstrumenter + + edu.gmu.swe.phosphor.jlink.JLinkRegistrationAgent + + edu.gmu.swe.phosphor.jlink.InstrumentDriver + true @@ -50,11 +77,33 @@ shade + + + org.jacoco.core + + edu.gmu.swe.phosphor.jlink.org.jacoco.core + + + + org.objectweb.asm + + edu.gmu.swe.phosphor.jlink.org.objectweb.asm + + + + + + org.jacoco:* + org.ow2.asm:* + + *:* + module-info.class + META-INF/*.SF META-INF/*.DSA META-INF/*.RSA @@ -62,6 +111,8 @@ false + true + true @@ -69,9 +120,9 @@ org.moditect moditect-maven-plugin - 1.0.0.RC2 + add-module-info package @@ -81,7 +132,7 @@ true - src/main/java/module-info.java + ${project.build.sourceDirectory}/module-info.java @@ -90,11 +141,4 @@ - - - edu.gmu.swe.phosphor - Phosphor - ${project.version} - - diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java index b26793847..35b63f5a6 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java @@ -1,14 +1,10 @@ package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; -import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.PhosphorOption; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.Option; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.Options; +import edu.columbia.cs.psl.phosphor.OptionsUtil; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; -import java.util.HashSet; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; @@ -20,8 +16,8 @@ public class JLinkInvoker { public static void invokeJLink(File jvmDir, File instJVMDir, Properties properties) { String jlinkBin = jvmDir + File.separator + "bin" + File.separator + "jlink"; File jlinkFile = getClassPathElement(JLinkInvoker.class); - String modulesToAdd = properties.getProperty(MODULES_PROPERTY, - "java.base,jdk.jdwp.agent,java.instrument,jdk.unsupported"); + String modulesToAdd = + properties.getProperty(MODULES_PROPERTY, "java.base,jdk.jdwp.agent,java.instrument,jdk.unsupported"); String classPath = buildClassPath(properties); ProcessBuilder pb = new ProcessBuilder( jlinkBin, @@ -32,8 +28,7 @@ public static void invokeJLink(File jvmDir, File instJVMDir, Properties properti "-J--class-path=" + classPath, "--output=" + instJVMDir, "--phosphor-transformer=transform" + createPhosphorJLinkPluginArgument(properties), - "--add-modules=" + modulesToAdd - ); + "--add-modules=" + modulesToAdd); try { System.out.println(String.join(" ", pb.command())); Process p = pb.inheritIO().start(); @@ -44,32 +39,7 @@ public static void invokeJLink(File jvmDir, File instJVMDir, Properties properti } private static String buildClassPath(Properties properties) { - Set> classes = new HashSet<>(); - Options options = PhosphorOption.createOptions(false); - for (Option option : options.getOptions()) { - if (option.getType().equals(Class.class)) { - String key = option.getOpt(); - if (properties.containsKey(key)) { - try { - Class clazz = Class.forName(properties.getProperty(key)); - classes.add(clazz); - } catch (ReflectiveOperationException e) { - String message = String.format("Failed to create %s class: %s", key, properties.getProperty(key)); - throw new IllegalArgumentException(message, e); - } - } - } - } - if (Configuration.PRIOR_CLASS_VISITOR != null) { - classes.add(Configuration.PRIOR_CLASS_VISITOR); - } - if (Configuration.POST_CLASS_VISITOR != null) { - classes.add(Configuration.POST_CLASS_VISITOR); - } - if (Configuration.taintTagFactoryPackage != null) { - classes.add(Configuration.taintTagFactory.getClass()); - } - return classes.stream() + return OptionsUtil.getConfigurationClasses(properties).stream() .map(JLinkInvoker::getClassPathElement) .map(File::getAbsolutePath) .collect(Collectors.joining(File.pathSeparator)); @@ -101,7 +71,8 @@ public static String createPhosphorJLinkPluginArgument(Properties properties) { public static File getClassPathElement(Class clazz) { try { - return new File(clazz.getProtectionDomain().getCodeSource().getLocation().toURI()); + return new File( + clazz.getProtectionDomain().getCodeSource().getLocation().toURI()); } catch (URISyntaxException e) { throw new AssertionError(); } diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java index 45b7df43c..4e2e8b77f 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java @@ -1,6 +1,7 @@ package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; import edu.columbia.cs.psl.phosphor.PhosphorPatcher; +import edu.gmu.swe.phosphor.jlink.InstrumentUtil; import jdk.tools.jlink.plugin.ResourcePool; import jdk.tools.jlink.plugin.ResourcePoolBuilder; import jdk.tools.jlink.plugin.ResourcePoolEntry; @@ -95,7 +96,7 @@ private void packJar(ResourcePoolBuilder out, File element, Set packages ZipEntry entry = entries.nextElement(); if (shouldInclude(entry.getName())) { try (InputStream is = zip.getInputStream(entry)) { - byte[] content = patcher.patch(entry.getName(), is.readAllBytes()); + byte[] content = patcher.patch(entry.getName(), InstrumentUtil.readAllBytes(is)); out.add(ResourcePoolEntry.create("/java.base/" + entry.getName(), content)); } packages.add(entry.getName().substring(0, entry.getName().lastIndexOf('/'))); diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/StandaloneJVMInstrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/StandaloneJVMInstrumenter.java deleted file mode 100644 index d446f66fb..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/StandaloneJVMInstrumenter.java +++ /dev/null @@ -1,35 +0,0 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - -import edu.columbia.cs.psl.phosphor.PhosphorOption; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.CommandLine; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.Option; - -import java.io.File; -import java.util.Properties; - -public class StandaloneJVMInstrumenter { - public static void main(String[] args) { - long START = System.currentTimeMillis(); - CommandLine line = PhosphorOption.configure(false, args); - if(line == null) { - return; - } - String[] remainingArgs = line.getArgs(); - File jvmDir = new File(remainingArgs[0]); - File instDir = new File(remainingArgs[1]); - Properties properties = toProperties(line); - JLinkInvoker.invokeJLink(jvmDir, instDir, properties); - } - - private static Properties toProperties(CommandLine line){ - Properties ret = new Properties(); - for(Option opt : line.getOptions()){ - String name = opt.getOpt(); - String value = line.getOptionValue(name); - if(value != null){ - ret.setProperty(name, value); - } - } - return ret; - } -} diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/DeletingFileVisitor.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java similarity index 94% rename from phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/DeletingFileVisitor.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java index 6dbd76590..53894db7f 100644 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/DeletingFileVisitor.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.ignored.maven; +package edu.gmu.swe.phosphor.jlink; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -11,7 +11,6 @@ * Deletes a file tree. */ public class DeletingFileVisitor extends SimpleFileVisitor { - /** * Deletes the specified file. * @param file {@inheritDoc} @@ -34,7 +33,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO */ @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { - if(e == null) { + if (e == null) { Files.delete(dir); return FileVisitResult.CONTINUE; } else { diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java new file mode 100644 index 000000000..296c3da35 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java @@ -0,0 +1,54 @@ +package edu.gmu.swe.phosphor.jlink; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public final class InstrumentDriver { + private InstrumentDriver() { + throw new AssertionError("Tried to instantiate static driver class: " + getClass()); + } + + /** + * Usage: java -cp <CLASS_PATH> edu.gmu.swe.phosphor.jlink.InstrumentDriver + * [OPTIONS] + * <INPUT_DIRECTORY> + * <OUTPUT_DIRECTORY> + * java -cp [CLASS-PATH] -jar phosphor.jar [OPTIONS] [input] [output] + * output_directory transformer_class core_jar [java_home] + *

+ * output_directory: directory to which the instrumented JVM should be written. + */ + public static void main(String[] args) throws IOException, ClassNotFoundException { + File inputDirectory = new File(args[0]); + File outputDirectory = new File(args[1]); + Class transformerClass = Class.forName(args[1]); + File coreJar = new File(args[2]); + File javaHome = args.length == 3 ? new File(System.getProperty("java.home")) : new File(args[3]); + System.out.printf("Instrumenting %s to %s%n", javaHome, outputDirectory); + long elapsedTime = instrument(inputDirectory, outputDirectory, null); + System.out.printf("Finished generation after %dms%n", elapsedTime); + } + + public static long instrument(File inputDirectory, File outputDirectory, Instrumentation instrumentation) + throws IOException { + if (outputDirectory.exists()) { + throw new IllegalArgumentException("Output directory exists."); + } + if (!inputDirectory.isDirectory()) { + throw new IllegalArgumentException("Invalid input directory: " + inputDirectory); + } + long startTime = System.currentTimeMillis(); + try { + if (InstrumentUtil.isModularJvm(inputDirectory)) { + // TODO + // Arrays.asList("--add-modules", "ALL-MODULE-PATH") + } else { + new Instrumenter(instrumentation).process(inputDirectory, outputDirectory); + } + return System.currentTimeMillis() - startTime; + } catch (IOException | InterruptedException | ExecutionException e) { + throw new IOException("Failed to generate instrumented JVM", e); + } + } +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java new file mode 100644 index 000000000..b7bfdf935 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java @@ -0,0 +1,128 @@ +package edu.gmu.swe.phosphor.jlink; + +import jdk.tools.jlink.plugin.Plugin; +import jdk.tools.jlink.plugin.ResourcePool; +import jdk.tools.jlink.plugin.ResourcePoolBuilder; +import jdk.tools.jlink.plugin.ResourcePoolEntry; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.commons.ModuleHashesAttribute; +import org.objectweb.asm.commons.ModuleResolutionAttribute; +import org.objectweb.asm.commons.ModuleTargetAttribute; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.ModuleExportNode; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.IllegalClassFormatException; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class InstrumentJLinkPlugin implements Plugin { + private ClassFileTransformer transformer; + private File coreJar; + + @Override + public String getName() { + return "phosphor-instrument"; + } + + @Override + public Category getType() { + return Category.FILTER; + } + + @Override + public String getDescription() { + return "Applies instrumentation to the runtime image and packs core classes into the java.base module"; + } + + @Override + public boolean hasArguments() { + return true; + } + + @Override + public void configure(Map config) { + coreJar = new File(config.get("core")); + try { + transformer = (ClassFileTransformer) Class.forName(config.get("transformer")) + .getDeclaredConstructor() + .newInstance(); + } catch (ReflectiveOperationException e) { + e.printStackTrace(); + throw new IllegalArgumentException("Failed to create transformer of type: " + config.get("transformer"), e); + } + } + + @Override + public ResourcePool transform(ResourcePool in, ResourcePoolBuilder out) { + in.transformAndCopy(e -> transform(e, out), out); + return out.build(); + } + + private ResourcePoolEntry transform(ResourcePoolEntry entry, ResourcePoolBuilder out) { + if (entry.type().equals(ResourcePoolEntry.Type.CLASS_OR_RESOURCE)) { + if (entry.path().endsWith("module-info.class") && entry.path().startsWith("/java.base")) { + // Transform java.base's module-info.class file and pack core classes into java.base + return entry.copyWithContent(transformBaseModuleInfo(entry.content(), out)); + } else if (entry.path().endsWith(".class")) { + try { + byte[] instrumented = transformer.transform(null, null, null, + null, entry.contentBytes()); + return instrumented == null ? entry : entry.copyWithContent(instrumented); + } catch (IllegalClassFormatException e) { + return entry; + } + } + } + return entry; + } + + private Set packCore(ResourcePoolBuilder out) throws IOException { + Set packages = new HashSet<>(); + try (ZipFile zip = new ZipFile(coreJar)) { + Enumeration entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.getName().endsWith(".class")) { + try (InputStream is = zip.getInputStream(entry)) { + out.add(ResourcePoolEntry.create("/java.base/" + entry.getName(), is.readAllBytes())); + } + packages.add(entry.getName().substring(0, entry.getName().lastIndexOf('/'))); + } + } + } + return packages; + } + + private byte[] transformBaseModuleInfo(InputStream in, ResourcePoolBuilder out) { + try { + Set packages = packCore(out); + ClassNode classNode = new ClassNode(); + ClassReader cr = new ClassReader(in); + Attribute[] attributes = new Attribute[]{new ModuleTargetAttribute(), + new ModuleResolutionAttribute(), + new ModuleHashesAttribute()}; + cr.accept(classNode, attributes, 0); + // Add exports + for (String packageName : packages) { + classNode.module.exports.add(new ModuleExportNode(packageName, 0, null)); + } + // Add packages + classNode.module.packages.addAll(packages); + ClassWriter cw = new ClassWriter(0); + classNode.accept(cw); + return cw.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java new file mode 100644 index 000000000..53b3aae6d --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java @@ -0,0 +1,87 @@ +package edu.gmu.swe.phosphor.jlink; + +import java.io.*; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public final class InstrumentUtil { + /** + * MD5 MessageDigest instance. + *
+ * Non-null. + */ + private static final MessageDigest md5Inst; + + static { + try { + md5Inst = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new ExceptionInInitializerError(e); + } + } + + private InstrumentUtil() { + throw new AssertionError(); + } + + public static File javaHomeToBin(File javaHome) { + return new File(javaHome, "bin"); + } + + public static File javaHomeToJavaExec(File javaHome) { + return new File(javaHomeToBin(javaHome), "java"); + } + + public static File javaHomeToJLinkExec(File javaHome) { + return new File(javaHomeToBin(javaHome), "jlink"); + } + + public static boolean isJavaHome(File directory) { + return javaHomeToJavaExec(directory).isFile(); + } + + public static boolean isModularJvm(File javaHome) { + return isJavaHome(javaHome) && javaHomeToJLinkExec(javaHome).isFile(); + } + + public static File getClassPathElement(Class clazz) { + return new File( + clazz.getProtectionDomain().getCodeSource().getLocation().getPath()); + } + + public static byte[] readAllBytes(File file) throws IOException { + try (FileInputStream in = new FileInputStream(file)) { + return readAllBytes(in); + } + } + + public static byte[] readAllBytes(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + copy(in, out); + return out.toByteArray(); + } + + public static void copy(InputStream in, OutputStream out) throws IOException { + byte[] buffer = new byte[1024]; + for (int len; (len = in.read(buffer)) != -1; ) { + out.write(buffer, 0, len); + } + } + + public static byte[] checksum(byte[] input) { + return md5Inst.digest(input); + } + + + /** + * Creates the specified directory if it does not already exist. + * + * @param dir the directory to create + * @throws IOException if the specified directory did not already exist and was not successfully created + */ + public static void ensureDirectory(File dir) throws IOException { + if (!dir.isDirectory() && !dir.mkdirs()) { + throw new IOException("Failed to create directory: " + dir); + } + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java new file mode 100644 index 000000000..6da2e80d1 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java @@ -0,0 +1,30 @@ +package edu.gmu.swe.phosphor.jlink; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; + +/** + * Implementors are expected to have a zero-argument, public constructor. + */ +public interface Instrumentation { + /** + * Configures this instance using the specified options. + * @param inputDirectory the directory to be instrumented + * @param outputDirectory the directory to which the instrumented output should be written + * @param options the key-value pairs that should be used to configure this instance + * @throws IOException if an I/O error occurs + */ + void configure(File inputDirectory, File outputDirectory, Properties options) + throws IOException, ReflectiveOperationException; + + /** + * Returns an array of class path elements needed to use this class. + * The returned array should be non-null. + * All elements of the returned array should be non-null. + * @return class path elements needed to use this class + */ + File[] getClassPathElements(); + + byte[] apply(byte[] classFileBuffer); +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java new file mode 100644 index 000000000..21eec919f --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java @@ -0,0 +1,186 @@ +package edu.gmu.swe.phosphor.jlink; + +import org.jacoco.core.internal.InputStreams; +import org.jacoco.core.internal.instr.SignatureRemover; + +import java.io.*; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.*; +import java.util.zip.*; + +public final class Instrumenter { + private final SignatureRemover signatureRemover = new SignatureRemover(); + private final ExecutorService executor = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + private final ConcurrentLinkedQueue errors = new ConcurrentLinkedQueue<>(); + private final Instrumentation instrumentation; + + public Instrumenter(Instrumentation instrumentation) { + if (instrumentation == null) { + throw new NullPointerException(); + } + this.instrumentation = instrumentation; + } + + public void process(File source, File target) throws IOException, InterruptedException, ExecutionException { + if (!source.exists()) { + throw new IllegalArgumentException("Source file not found: " + source); + } else if (!source.isDirectory() && !isClass(source.getName()) && !isArchive(source.getName())) { + throw new IllegalArgumentException("Unknown source file type: " + source); + } + Queue> futures = new LinkedList<>(); + processFile(futures, source, target); + while (!futures.isEmpty()) { + futures.poll().get(); + } + executor.shutdown(); + while (!executor.isTerminated()) { + if (executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS)) { + break; + } + } + if (!errors.isEmpty()) { + for (Throwable error : errors) { + error.printStackTrace(); + } + } + } + + private void instrumentClass(File source, File target) { + try (InputStream input = Files.newInputStream(source.toPath()); + OutputStream output = Files.newOutputStream(target.toPath())) { + instrumentClass(input, output); + } catch (Throwable t) { + errors.add(t); + } + } + + private void instrumentClass(InputStream input, OutputStream output) throws IOException { + byte[] buffer = InstrumentUtil.readAllBytes(input); + byte[] result = instrumentation.apply(buffer); + output.write(result == null ? buffer : result); + } + + private void processFile(Collection> futures, File source, File target) + throws IOException, InterruptedException { + if (source.isDirectory()) { + InstrumentUtil.ensureDirectory(target); + for (File child : Objects.requireNonNull(source.listFiles())) { + processFile(futures, child, new File(target, child.getName())); + } + } else if (isClass(source.getName())) { + futures.add(executor.submit(() -> instrumentClass(source, target), null)); + } else if (isArchive(source.getName())) { + processZip(Files.newInputStream(source.toPath()), Files.newOutputStream(target.toPath())); + } else { + if (copy(source, target)) { + if (source.canExecute() && !target.setExecutable(true)) { + errors.add(new IOException("Failed to set permissions for: " + target)); + } + if (source.canRead() && !target.setReadable(true)) { + errors.add(new IOException("Failed to set permissions for: " + target)); + } + if (source.canWrite() && !target.setWritable(true)) { + errors.add(new IOException("Failed to set permissions for: " + target)); + } + } + } + } + + private void processZip(InputStream in, OutputStream out) throws IOException, InterruptedException { + try { + List> futures = new LinkedList<>(); + try (ZipInputStream zin = new ZipInputStream(in)) { + for (ZipEntry entry; (entry = zin.getNextEntry()) != null; ) { + ZipEntry finalEntry = entry; + if (entry.isDirectory()) { + futures.add(executor.submit(() -> new ZipResult(finalEntry, null))); + } else if (!signatureRemover.removeEntry(entry.getName())) { + byte[] buffer = InputStreams.readFully(zin); + futures.add(executor.submit(() -> new ZipResult(finalEntry, buffer))); + } + } + } + writeZipResults(out, futures); + } catch (IOException e) { + errors.add(e); + InstrumentUtil.copy(in, out); + } + } + + private void writeZipResults(OutputStream out, List> futures) + throws IOException, InterruptedException { + try (ZipOutputStream zos = new ZipOutputStream(out)) { + for (Future future : futures) { + try { + ZipResult result = future.get(); + ZipEntry entry = result.entry; + ZipEntry outEntry = new ZipEntry(entry.getName()); + outEntry.setMethod(entry.getMethod()); + if (entry.getMethod() == ZipEntry.STORED) { + // Uncompressed entries require entry size and CRC + outEntry.setSize(result.buffer.length); + outEntry.setCompressedSize(result.buffer.length); + CRC32 crc = new CRC32(); + crc.update(result.buffer, 0, result.buffer.length); + outEntry.setCrc(crc.getValue()); + } + zos.putNextEntry(outEntry); + zos.write(result.buffer); + zos.closeEntry(); + } catch (ExecutionException | ZipException e) { + errors.add(e); + } + } + zos.finish(); + } + } + + private boolean copy(File source, File target) { + try (InputStream in = Files.newInputStream(source.toPath()); + OutputStream out = Files.newOutputStream(target.toPath())) { + InstrumentUtil.copy(in, out); + return true; + } catch (IOException e) { + errors.add(e); + return false; + } + } + + private static boolean isArchive(String name) { + return name.endsWith(".jar") || name.endsWith(".war") || name.endsWith(".zip") || name.endsWith(".jmod"); + } + + private static boolean isClass(String name) { + return name.endsWith(".class"); + } + + private class ZipResult { + private final ZipEntry entry; + private final byte[] buffer; + + public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedException { + this.entry = entry; + byte[] tempBuffer = new byte[0]; + if (buffer != null) { + tempBuffer = buffer; + try { + ByteArrayInputStream in = new ByteArrayInputStream(buffer); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + if (entry.getName().endsWith(".class")) { + instrumentClass(in, out); + } else if (entry.getName().endsWith(".jar")) { + processZip(in, out); + } else if (!signatureRemover.filterEntry(entry.getName(), in, out)) { + InstrumentUtil.copy(in, out); + } + tempBuffer = out.toByteArray(); + } catch (IOException | RuntimeException e) { + errors.add(e); + } + } + this.buffer = tempBuffer; + } + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java new file mode 100644 index 000000000..e3c87ef2e --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java @@ -0,0 +1,68 @@ +package edu.gmu.swe.phosphor.jlink; + +import edu.columbia.cs.psl.phosphor.OptionsUtil; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +public final class JLinkInvoker { + private JLinkInvoker() { + throw new AssertionError(); + } + + public static void invoke( + File javaHome, File outputDirectory, List jlinkOptions, Class transformerClass, File coreJar) + throws InterruptedException, IOException { + Properties properties = new Properties(); + String jlinkAgentJar = + InstrumentUtil.getClassPathElement(JLinkRegistrationAgent.class).getAbsolutePath(); + List command = new ArrayList<>(); + command.add(InstrumentUtil.javaHomeToJLinkExec(javaHome).getAbsolutePath()); + command.add("-J-javaagent:" + jlinkAgentJar); + command.add("-J--class-path=" + buildClassPath(transformerClass, properties)); + command.add("-J--add-reads=" + JLinkRegistrationAgent.MODULE_NAME + "=ALL-UNNAMED"); + command.add("-J--module-path=" + jlinkAgentJar); + command.add("-J--add-modules=" + JLinkRegistrationAgent.MODULE_NAME); + command.add(getPluginOption(transformerClass, coreJar)); + command.add("--output=" + outputDirectory.getAbsolutePath()); + command.addAll(processJLinkOptions(jlinkOptions)); + ProcessBuilder builder = new ProcessBuilder(command); + System.out.println(String.join(" ", builder.command())); + Process process = builder.inheritIO().start(); + if (process.waitFor() != 0) { + throw new RuntimeException("Failed to create instrumented runtime image"); + } + } + + private static String buildClassPath(Class transformerClass, Properties properties) { + Set> classes = OptionsUtil.getConfigurationClasses(properties); + classes.add(transformerClass); + return classes.stream() + .map(edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.JLinkInvoker::getClassPathElement) + .map(File::getAbsolutePath) + .collect(Collectors.joining(File.pathSeparator)); + } + + private static List processJLinkOptions(List jlinkOptions) { + // Ensure that all required modules are included + jlinkOptions = new ArrayList<>(jlinkOptions); + Set modules = + new LinkedHashSet<>(Arrays.asList("java.base", "jdk.jdwp.agent", "java.instrument", "jdk.unsupported")); + int i = jlinkOptions.indexOf("--add-modules"); + if (i != -1) { + modules.addAll(Arrays.asList(jlinkOptions.get(i + 1).split(","))); + jlinkOptions.set(i + 1, String.join(",", modules)); + } else { + jlinkOptions.add("--add-modules"); + jlinkOptions.add(String.join(",", modules)); + } + return jlinkOptions; + } + + private static String getPluginOption(Class transformerClass, File coreJar) { + return String.format( + "--phosphor-instrument=x:transformer=%s:core=%s", transformerClass.getName(), coreJar.getAbsolutePath()); + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java new file mode 100644 index 000000000..845b3f59f --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java @@ -0,0 +1,247 @@ +/* + * Based on the JLinkPluginRegistrationAgent class from hibernate-demos (https://github.com/hibernate/hibernate-demos) + * which is licensed under the following terms: + * + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "{}" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright {yyyy} {name of copyright owner} + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.gmu.swe.phosphor.jlink; + +import java.lang.instrument.Instrumentation; +import java.util.*; + +/** + * Alters the jdk.jlink module to export the "jdk.tools.jlink.plugin" package to this class' module. + * Alters this class' module to provide our custom JLink plugin as a service. + */ +@SuppressWarnings("Java9ReflectionClassVisibility") +public final class JLinkRegistrationAgent { + public static final String MODULE_NAME = "edu.gmu.swe.phosphor.instrument"; + private static final String JLINK_MODULE_NAME = "jdk.jlink"; + private static final String JLINK_PLUGIN_PACKAGE_NAME = "jdk.tools.jlink.plugin"; + private static final String JLINK_PLUGIN_CLASS_NAME = JLINK_PLUGIN_PACKAGE_NAME + ".Plugin"; + + public static void premain(String agentArgs, Instrumentation inst) throws Exception { + Module jlinkModule = ModuleLayer.boot().findModule(JLINK_MODULE_NAME).orElseThrow(RuntimeException::new); + Module customPluginModule = + ModuleLayer.boot().findModule(MODULE_NAME).orElseThrow(RuntimeException::new); + Map> extraExports = new HashMap<>(); + extraExports.put(JLINK_PLUGIN_PACKAGE_NAME, Collections.singleton(customPluginModule)); + // Alter jdk.jlink to export the Plugin API to the module containing our custom JLink plugin + inst.redefineModule(jlinkModule, + Collections.emptySet(), + extraExports, + Collections.emptyMap(), + Collections.emptySet(), + Collections.emptyMap()); + Class jlinkPluginClass = jlinkModule.getClassLoader().loadClass(JLINK_PLUGIN_CLASS_NAME); + Map, List>> extraProvides = new HashMap<>(); + extraProvides.put(jlinkPluginClass, Collections.singletonList(InstrumentJLinkPlugin.class)); + // Alter our custom JLink plugin's module to provide our custom JLink plugin as a service + inst.redefineModule(customPluginModule, + Collections.emptySet(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), + extraProvides); + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java new file mode 100644 index 000000000..4f7d211b0 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java @@ -0,0 +1,35 @@ +package edu.gmu.swe.phosphor.jlink; + +import edu.columbia.cs.psl.phosphor.Configuration; +import edu.columbia.cs.psl.phosphor.OptionsUtil; +import edu.columbia.cs.psl.phosphor.PhosphorOption; +import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; + +import java.io.File; +import java.util.Properties; + +public class PhosphorInstrumentation implements Instrumentation { + private PreMain.PCLoggingTransformer transformer; + + @Override + public void configure(File inputDirectory, File outputDirectory, Properties options) { + String[] arguments = OptionsUtil.createPhosphorMainArguments(inputDirectory, outputDirectory, options); + PhosphorOption.configure(false, arguments); + Configuration.init(); + TaintTrackingClassVisitor.IS_RUNTIME_INST = false; + transformer = new PreMain.PCLoggingTransformer(); + } + + @Override + public File[] getClassPathElements() { + return new File[] {InstrumentUtil.getClassPathElement(PreMain.class)}; + } + + @Override + public byte[] apply(byte[] classFileBuffer) { + transformer.transform(null, null, null, null, + classFileBuffer, false); + return null; + } +} diff --git a/phosphor-instrument-jigsaw/src/main/java/module-info.java b/phosphor-instrument-jigsaw/src/main/java/module-info.java index 5361cd48f..5d368d70a 100644 --- a/phosphor-instrument-jigsaw/src/main/java/module-info.java +++ b/phosphor-instrument-jigsaw/src/main/java/module-info.java @@ -1,23 +1,5 @@ -module edu.columbia.cs.psl.jigsaw.phosphor.instrumenter { - exports edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - opens edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - opens edu.columbia.cs.psl.phosphor.instrumenter.analyzer; - opens edu.columbia.cs.psl.phosphor.struct.harmony.util; - opens edu.columbia.cs.psl.phosphor.control.graph; - opens edu.columbia.cs.psl.phosphor.instrumenter; - opens edu.columbia.cs.psl.phosphor.runtime.jdk.unsupported; - opens edu.columbia.cs.psl.phosphor.control.type; - opens edu.columbia.cs.psl.phosphor.org.objectweb.asm.tree; - opens edu.columbia.cs.psl.phosphor.org.objectweb.asm.commons; - opens edu.columbia.cs.psl.phosphor.struct; - opens edu.columbia.cs.psl.phosphor.control.standard; - opens edu.columbia.cs.psl.phosphor.instrumenter.asm; - opens edu.columbia.cs.psl.phosphor.control; - opens edu.columbia.cs.psl.phosphor.org.objectweb.asm.analysis; - opens edu.columbia.cs.psl.phosphor; - opens edu.columbia.cs.psl.phosphor.org.objectweb.asm; - opens edu.columbia.cs.psl.phosphor.runtime; - opens edu.columbia.cs.psl.phosphor.runtime.proxied; +module edu.gmu.swe.phosphor.jlink { + exports edu.gmu.swe.phosphor.jlink; requires jdk.jlink; requires java.instrument; } diff --git a/phosphor-instrument-maven-plugin/pom.xml b/phosphor-instrument-maven-plugin/pom.xml index ec7fbbded..9cf17a6c9 100644 --- a/phosphor-instrument-maven-plugin/pom.xml +++ b/phosphor-instrument-maven-plugin/pom.xml @@ -3,98 +3,22 @@ 4.0.0 phosphor-instrument-maven-plugin maven-plugin - A Maven Plugin for Creating Phosphor-instrumented JVMs + A Maven plugin for creating phosphor-instrumented JVMs edu.gmu.swe.phosphor phosphor-parent 0.1.0-SNAPSHOT - - - - maven-compiler-plugin - 3.8.0 - - 1.8 - 1.8 - true - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.1 - - true - - - - attach-javadoc - - jar - - - - - - org.apache.maven.plugins - maven-plugin-plugin - 3.6.0 - - - mojo-descriptor - - descriptor - - - - - - - - - release-sign-artifacts - - - gpg.passphrase - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - - --pinentry-mode - loopback - - - - - - - org.apache.maven maven-plugin-api - 3.6.2 + 3.8.1 org.apache.maven.plugin-tools maven-plugin-annotations - 3.6.0 + 3.6.1 provided @@ -108,4 +32,21 @@ ${project.version} + + + + org.apache.maven.plugins + maven-plugin-plugin + 3.6.0 + + + mojo-descriptor + + descriptor + + + + + + diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/ExecutePermissionAssigningFileVisitor.java b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/ExecutePermissionAssigningFileVisitor.java deleted file mode 100644 index d0ff607ba..000000000 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/ExecutePermissionAssigningFileVisitor.java +++ /dev/null @@ -1,28 +0,0 @@ -package edu.gmu.swe.phosphor.ignored.maven; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -/** - * Traverses a file tree. Assigns execute permission to visited files. - */ -public class ExecutePermissionAssigningFileVisitor extends SimpleFileVisitor { - - /** - * Assigns execute permission to the specified file. - * @param file {@inheritDoc} - * @param attrs {@inheritDoc} - * @return FileVisitResult#CONTINUE - * @throws IOException if assigning execute permission fails - */ - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if(!file.toFile().setExecutable(true)) { - throw new IOException("Failed to assign execute permission to: " + file); - } - return FileVisitResult.CONTINUE; - } -} diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentUtil.java b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentUtil.java deleted file mode 100644 index ae2c58c1f..000000000 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentUtil.java +++ /dev/null @@ -1,184 +0,0 @@ -package edu.gmu.swe.phosphor.ignored.maven; - -import edu.columbia.cs.psl.phosphor.Instrumenter; -import edu.columbia.cs.psl.phosphor.PhosphorOption; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.Option; -import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.Options; -import edu.columbia.cs.psl.phosphor.struct.SinglyLinkedList; -import org.codehaus.plexus.util.FileUtils; - -import java.io.File; -import java.io.IOException; -import java.lang.instrument.Instrumentation; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.security.MessageDigest; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -/** - * Utility class that provides methods to assist with Phosphor instrumentation. - */ -public class PhosphorInstrumentUtil { - - /** - * MD5 MessageDigest instance used for generating checksums. - */ - private static MessageDigest md5Inst; - - static { - try { - md5Inst = MessageDigest.getInstance("MD5"); - } catch(Exception e) { - System.err.println("Failed to create MD5 MessageDigest"); - e.printStackTrace(); - } - } - - private PhosphorInstrumentUtil() { - - } - - /** - * Creates a standardized copy of the specified properties where each Phosphor option without an argument is mapped - * to either true or not present in the properties, each Phosphor option with an argument is either mapped to a - * non-null, non-empty string or not present in properties, and no other keys are present in the properties. - * - * @param isRuntimeInst true if the options should be standardized against the options available during dynamic - * instrumentation, otherwise standardizes against the options available during static - * instrumentation - * @param properties un-standardized properties to be standardized - * @return a standardized copy of properties - */ - public static Properties canonicalizeProperties(Properties properties, boolean isRuntimeInst) { - Set propNames = properties.stringPropertyNames(); - Properties canonicalProps = new Properties(); - Map phosphorOptionMap = createPhosphorOptionMap(isRuntimeInst); - for(String propName : propNames) { - if(phosphorOptionMap.containsKey(propName)) { - Option option = phosphorOptionMap.get(propName); - if(option.hasArg()) { - if(properties.getProperty(propName) != null && properties.getProperty(propName).length() > 0) { - canonicalProps.setProperty(option.getOpt(), properties.getProperty(propName)); - } - } else { - if(properties.getProperty(propName).length() == 0 || "true".equals(properties.getProperty(propName).toLowerCase())) { - canonicalProps.setProperty(option.getOpt(), "true"); - } - } - } else { - System.err.println("Unknown Phosphor option: " + propName); - } - } - return canonicalProps; - } - - /** - * @param isRuntimeInst true if a map of options available during dynamic instrumentation should be returned, - * otherwise a map of option available during static instrumentation should be returned - * @return a mapping from the names of configuration options available in Phosphor to an instance of - * org.apache.commons.cli.Option that represents that configuration option - */ - public static Map createPhosphorOptionMap(boolean isRuntimeInst) { - Map phosphorOptionMap = new HashMap(); - Options options = PhosphorOption.createOptions(isRuntimeInst); - for(Option option : options.getOptions()) { - phosphorOptionMap.put(option.getOpt(), option); - if(option.hasLongOpt()) { - phosphorOptionMap.put(option.getLongOpt(), option); - } - } - return phosphorOptionMap; - } - - /** - * @return a File object pointing to the JAR file for Phosphor - */ - public static File getPhosphorJarFile() { - try { - return new File(Instrumenter.class.getProtectionDomain().getCodeSource().getLocation().toURI()); - } catch(URISyntaxException e) { - throw new AssertionError(); - } - } - - /** - * @return the checksum of the JAR file for Phosphor - */ - public static byte[] generateChecksumForPhosphorJar() throws IOException { - byte[] jarBytes = Files.readAllBytes(PhosphorInstrumentUtil.getPhosphorJarFile().toPath()); - return md5Inst.digest(jarBytes); - } - - /** - * Ensures that the specified directory exists and is empty. - * - * @param dir the directory to be created or cleaned - * @throws IOException if the specified directory could not be created or cleaned - */ - public static void createOrCleanDirectory(File dir) throws IOException { - if(dir.isDirectory()) { - FileUtils.cleanDirectory(dir); - } else { - if(dir.isFile()) { - if(!dir.delete()) { - throw new IOException("Failed to delete: " + dir); - } - } - if(!dir.mkdirs()) { - throw new IOException("Failed to create directory: " + dir); - } - } - } - - /** - * @param properties canonicalized properties that specify the Phosphor configuration options that should set in the - * created argument - * @return a String formatted for {@link edu.columbia.cs.psl.phosphor.PreMain#premain(String, Instrumentation) premain's} - * String argument - */ - public static String createPhosphorAgentArgument(Properties properties) { - if(properties.isEmpty()) { - return ""; - } else { - StringBuilder builder = new StringBuilder().append("="); - Set propNames = properties.stringPropertyNames(); - boolean first = true; - for(String propName : propNames) { - if(first) { - first = false; - } else { - builder.append(','); - } - builder.append(propName); - if(!"true".equals(properties.getProperty(propName))) { - builder.append('=').append(properties.getProperty(propName)); - } - } - return builder.toString(); - } - } - - /** - * @param jvmDir the source directory where the JDK or JRE is installed - * @param instJVMDir the target directory where the Phosphor-instrumented JVM should be created - * @param properties canonicalized properties that specify the Phosphor configuration options that should set in the - * created arguments - * @return an array formatted for {@link Instrumenter#main(String[])} Instrumenter.main's} String[] argument - */ - public static String[] createPhosphorMainArguments(File jvmDir, File instJVMDir, Properties properties) { - SinglyLinkedList arguments = new SinglyLinkedList<>(); - Set propNames = properties.stringPropertyNames(); - for(String propName : propNames) { - arguments.addLast("-" + propName); - if(!"true".equals(properties.getProperty(propName))) { - arguments.addLast(properties.getProperty(propName)); - } - } - arguments.addLast(jvmDir.getAbsolutePath()); - arguments.addLast(instJVMDir.getAbsolutePath()); - return arguments.toArray(new String[0]); - } -} diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentingMojo.java b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentingMojo.java deleted file mode 100644 index 18282b538..000000000 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/ignored/maven/PhosphorInstrumentingMojo.java +++ /dev/null @@ -1,258 +0,0 @@ -package edu.gmu.swe.phosphor.ignored.maven; - -import edu.columbia.cs.psl.phosphor.Instrumenter; -import edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.JLinkInvoker; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.plugins.annotations.LifecyclePhase; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; - -import static edu.gmu.swe.phosphor.ignored.maven.PhosphorInstrumentUtil.createPhosphorMainArguments; -import static edu.gmu.swe.phosphor.ignored.maven.PhosphorInstrumentUtil.generateChecksumForPhosphorJar; - -/** - * Creates a Phosphor-instrumented JVM at a target location. If a directory already exists at the target location, - * checks to see if it contains a phosphor-properties file and phosphor-checksum.md5sum file. If it does and the - * phosphor-properties file's properties match the desired properties for the Phosphor-instrumented JVM to be created, - * and the phosphor-checksum.md5sum file's checksum matches the checksum of the Phosphor JAR current loaded, this - * directory is assumed to contain a properly instrumented JVM. Otherwise, deletes any existing files or directories at - * the target location and creates a new Phosphor-instrumented JVM there. - *

- * If a new Phosphor-instrumented JVM is generated, deletes any directories listed as being used to cache files - * dynamically instrumented by Phosphor when using the previous Phosphor-instrumented JVM. - */ -@Mojo(name = "instrument", defaultPhase = LifecyclePhase.COMPILE) -public class PhosphorInstrumentingMojo extends AbstractMojo { - - /** - * Constant name of the file used to store information about the configuration options that were used by Phosphor - * when instrumenting a JVM. - */ - public static final String PROPERTIES_FILE_NAME = "phosphor-properties"; - - /** - * Constant name of the file used to store the checksum of the Phosphor JAR used to instrument a JVM. - */ - public static final String CHECKSUM_FILE_NAME = "phosphor-checksum.md5sum"; - - /** - * Path to directory where the JDK or JRE installation to be instrumented is installed. - */ - @Parameter(property = "sourceJVMDir", readonly = true) - private String sourceJVMDir; - - /** - * Path to a directory in which the target location for the instrumented JVM should be placed. - */ - @Parameter(property = "targetBaseDir", defaultValue = "${project.build.directory}", readonly = true) - private File targetBaseDir; - - /** - * Name of the target location directory for the instrumented JVM. - */ - @Parameter(property = "instrumentedJVMName", readonly = true, required = true) - private String targetName; - - /** - * Phosphor configuration options that should be set while instrumenting the JVM. - */ - @Parameter(property = "options", readonly = true, required = true) - private Properties options; - - /** - * True if an existing instrumented JVM should be invalidated and rebuilt if it was instrumented with a Phosphor - * JAR file whose checksum does not match the current Phosphor JAR file's checksum - */ - @Parameter(property = "reinstrumentBasedOnChecksum", readonly = true, defaultValue = "true") - private boolean reinstrumentBasedOnChecksum; - - /** - * List of paths to directories that are used to cache files dynamically instrumented by Phosphor that - * should be delete if a new Phosphor-instrumented JVM is generated - */ - @Parameter(property = "associatedCaches", readonly = true) - private List associatedCaches; - - /** - * Creates a Phosphor-instrumented JVM at a target location if one does not already exist with the correct - * properties and checksum. - * - * @throws MojoFailureException if the Phosphor-instrumented JVM cannot be created at the target location - */ - @Override - public void execute() throws MojoFailureException { - Properties canonicalOptions = PhosphorInstrumentUtil.canonicalizeProperties(options, false); - File jvmDir = getJVMDir(sourceJVMDir); - File instJVMDir = new File(targetBaseDir, targetName); - byte[] checksum; - try { - checksum = generateChecksumForPhosphorJar(); - } catch(IOException e) { - throw new MojoFailureException("Failed to generate checksum for Phosphor JAR"); - } - if(Instrumenter.isJava8JVMDir(jvmDir)){ - canonicalOptions.setProperty("java8", "true"); - } - if(!isExistingInstrumentedJVM(instJVMDir, canonicalOptions, checksum)) { - getLog().info(String.format("Generating Phosphor-instrumented JVM %s with options: %s", instJVMDir, canonicalOptions)); - try { - generateInstrumentedJVM(jvmDir, instJVMDir, canonicalOptions, checksum); - } catch(IOException e) { - throw new MojoFailureException("Failed to create instrumented JVM", e); - } - // Delete associated caches - for(String associateCache : associatedCaches) { - if(associateCache != null) { - deleteCache(new File(associateCache)); - } - } - } else { - String message = String.format("No generation necessary: existing Phosphor-instrumented JVM %s with " + - "correct properties(%s)%s found", instJVMDir, canonicalOptions, - reinstrumentBasedOnChecksum ? " and checksum" : ""); - getLog().info(message); - } - } - - /** - * Checks whether the specified file is a directory containing a phosphor-properties file whose properties match - * the specified desired properties and a phosphor-checksum file whose bytes match the specified checksum - * (if invalidating invalidateBasedOnChecksum is true). - * - * @param instJVMDir path to be checked for an existing instrumented JVM - * @param desiredProperties the properties that the existing instrumented JVM must have - * @param checksum the checksum that existing instrumented JVM must have associated with it - * @return true if instJVMDir is a directory that contains a phosphor-properties file that matches desiredProperties - * and a phosphor-checksum file whose bytes match the specified checksum (if invalidating - * invalidateBasedOnChecksum is true). - */ - private boolean isExistingInstrumentedJVM(File instJVMDir, Properties desiredProperties, byte[] checksum) { - if(instJVMDir.isDirectory()) { - File propFile = new File(instJVMDir, PROPERTIES_FILE_NAME); - if(propFile.isFile()) { - try { - Properties existingProperties = new Properties(); - existingProperties.load(new FileReader(propFile)); - if(!desiredProperties.equals(existingProperties)) { - return false; - } - } catch(IOException e) { - return false; - } - } - if(reinstrumentBasedOnChecksum) { - try { - byte[] existingChecksum = Files.readAllBytes(new File(instJVMDir, CHECKSUM_FILE_NAME).toPath()); - return Arrays.equals(checksum, existingChecksum); - } catch(IOException e) { - return false; - } - } else { - return true; - } - } - return false; - } - - /** - * Tries to delete the specified cache directory. Logs a warning if deletion is unsuccessful. - * - * @param cacheDir directory containing a cache files dynamically instrumented by Phosphor - */ - private void deleteCache(File cacheDir) { - if(cacheDir.exists()) { - getLog().info("Deleting associated Phosphor cache directory: " + cacheDir); - try { - Files.walkFileTree(cacheDir.toPath(), new DeletingFileVisitor()); - } catch(IOException e) { - getLog().warn("Failed to delete associated Phosphor cache directory: " + cacheDir); - getLog().warn(e); - } - } - } - - /** - * Determines the correct path for the directory where the JDK or JRE installation to be instrumented is installed - * as follows: - * Use the value of baseJVM if it provided and is non-null and not the empty string. Else use the value of the - * environmental variable INST_HOME if it is set to a non-null, non empty string value. Else use the value of - * the environmental variable JAVA_HOME if it is set to a non-null, non empty string value. Otherwise no appropriate - * path value was provided. - * - * @param sourceJVMDir if non-null and non-empty specified the location of the JDK or JRE installation to be - * instrumented - * @throws MojoFailureException if an appropriate JDK or JRE installation path was not provided or the provided - * path does not point to a directory - * @ return a file representing the path to the determined JDK or JRE installation directory - */ - private static File getJVMDir(String sourceJVMDir) throws MojoFailureException { - String path; - String source; - if(sourceJVMDir != null && !sourceJVMDir.isEmpty()) { - path = sourceJVMDir; - source = "sourceJVMDir property"; - } else if(System.getenv("INST_HOME") != null && !System.getenv("INST_HOME").isEmpty()) { - path = System.getenv("INST_HOME"); - source = "INST_HOME environmental variable"; - } else if(System.getenv("JAVA_HOME") != null && !System.getenv("JAVA_HOME").isEmpty()) { - path = System.getenv("JAVA_HOME"); - source = "JAVA_HOME environmental variable"; - } else { - throw new MojoFailureException("Either baseJVM property or INST_HOME environmental variable or JAVA_HOME " + - "environmental variable must be set to a directory where the JDK or JRE is installed"); - } - File jvmDir = new File(path); - if(!jvmDir.isDirectory()) { - throw new MojoFailureException(String.format("Value for %s (%s) must be set to a directory where the " + - "JDK or JRE is installed", source, path)); - } - return jvmDir; - } - - /** - * Creates a Phosphor-instrumented JVM at the specified directory with the specified Phosphor configuration options. - * - * @param jvmDir the source directory where the JDK or JRE is installed - * @param instJVMDir the target directory where the Phosphor-instrumented JVM should be created - * @param properties canonicalized properties that specify the Phosphor configuration options that should be used - * @param checksum the checksum of the Phosphor JAR that will be used to instrument the JVM - * @throws IOException if an I/O error occurs - */ - private static void generateInstrumentedJVM(File jvmDir, File instJVMDir, Properties properties, byte[] checksum) throws IOException { - if(instJVMDir.exists()) { - // Delete existing directory or file if necessary - Files.walkFileTree(instJVMDir.toPath(), new DeletingFileVisitor()); - } - if(Instrumenter.isJava8JVMDir(jvmDir)){ - if(!instJVMDir.mkdirs()) { - throw new IOException("Failed to create target directory for Phosphor-instrumented JVM: " + instJVMDir); - } - Instrumenter.main(createPhosphorMainArguments(jvmDir, instJVMDir, properties)); - } else{ - JLinkInvoker.invokeJLink(jvmDir, instJVMDir, properties); - } - // Add phosphor-properties file to directory - File propsFile = new File(instJVMDir, PROPERTIES_FILE_NAME); - properties.store(new FileWriter(propsFile), null); - // Add phosphor-checksum file to directory - File checksumFile = new File(instJVMDir, CHECKSUM_FILE_NAME); - Files.write(checksumFile.toPath(), checksum); - // Set execute permissions - Files.walkFileTree(new File(instJVMDir, "bin").toPath(), new ExecutePermissionAssigningFileVisitor()); - Files.walkFileTree(new File(instJVMDir, "lib").toPath(), new ExecutePermissionAssigningFileVisitor()); - if(new File(instJVMDir, "jre").exists()) { - Files.walkFileTree(new File(instJVMDir, "jre" + File.separator + "bin").toPath(), new ExecutePermissionAssigningFileVisitor()); - Files.walkFileTree(new File(instJVMDir, "jre" + File.separator + "lib").toPath(), new ExecutePermissionAssigningFileVisitor()); - } - } -} diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java new file mode 100644 index 000000000..c2f594c40 --- /dev/null +++ b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java @@ -0,0 +1,221 @@ +package edu.gmu.swe.phosphor.instrument; + +import edu.gmu.swe.phosphor.jlink.DeletingFileVisitor; +import edu.gmu.swe.phosphor.jlink.InstrumentDriver; +import edu.gmu.swe.phosphor.jlink.InstrumentUtil; +import edu.gmu.swe.phosphor.jlink.Instrumentation; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + +import java.io.*; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +/** + * Creates an instrumented Java installation. + *

+ * If {@link InstrumentingMojo#outputDirectory} does not exists, a new instrumented Java installation is created. + * If {@link InstrumentingMojo#outputDirectory} exists and is not a Java installation previously created by + * this plugin, this plugin will throw a {@link MojoExecutionException}. + * If {@link InstrumentingMojo#outputDirectory} exists and is a Java installation previously created by this plugin, + * this plugin checks whether the Java installation needs to be recreated. + * If the {@link InstrumentingMojo#forceCreation} is {@code false} and the existing Java installation was created from + * the same {@link InstrumentingMojo#javaHome uninstrumented Java installation} using the same + * {@link InstrumentingMojo#instrumentationType} and {@link InstrumentingMojo#options}, then the Java installation does + * not need to be recreated and the plugin terminates. + * Otherwise, this plugin will delete the existing Java installation and any + * {@link InstrumentingMojo#linkedCaches linked files or directories}. + * Then, this plugin will create a new instrumented Java installation. + *

+ * The instrumented Java installation is created by instrumenting the Java installation located in the directory + * {@link InstrumentingMojo#javaHome} or the Java installation used to run the Maven process if + * {@link InstrumentingMojo#javaHome} was not specified. + * An instance of the specified {@link InstrumentingMojo#instrumentationType instrumentation class} is created and + * configured using the specified {@link InstrumentingMojo#options}. + * This instance determines the type of instrumentation applied. + * + * @see edu.gmu.swe.phosphor.jlink.Instrumentation + */ +@Mojo(name = "instrument", defaultPhase = LifecyclePhase.PROCESS_TEST_RESOURCES) +public class InstrumentingMojo extends AbstractMojo { + /** + * Directory where the Java installation to be instrumented is located. + * If not specified, then the Java installation used to run the Maven process will be used. + */ + @Parameter(property = "phosphor.javaHome") + private File javaHome = new File(System.getProperty("java.home")); + /** + * Directory to which the instrumented Java installation should be written. + */ + @Parameter(property = "phosphor.outputDirectory", defaultValue = "${project.build.directory}/phosphor/java/") + private File outputDirectory; + /** + * List of directories and files that should be deleted if an existing instrumented Java installation is deleted. + * These can be used to specify caches of dynamically instrumented classes. + */ + @Parameter(property = "phosphor.linkedCaches") + private List linkedCaches = new LinkedList<>(); + /** + * True if a new instrumented Java installation should be created even if a Java installation instrumented by this + * plugin exists and was created using the same settings that would be used by this plugin. + */ + @Parameter(property = "phosphor.forceCreation", readonly = true, defaultValue = "false") + private boolean forceCreation; + /** + * Fully qualified name of the implementation of {@link Instrumentation} that should be used to instrument the + * Java installation. + * This plugin must be configured to add the the artifact that contains this implementation as a dependency using + * the "dependencies" Maven tag for plugins. + * + * @see + * Using the Dependencies Tag + * + * @see Class#forName(String className) + */ + @Parameter( + property = "phosphor.instrumentationType", + defaultValue = "edu.gmu.swe.phosphor.jlink.PhosphorInstrumentation") + private String instrumentationType; + /** + * Options passed to {@link Instrumentation#configure}. + */ + @Parameter(property = "phosphor.options") + private Properties options = new Properties(); + /** + * File used to store the options used by this plugin to create the instrumented Java installation + */ + private File optionsFile; + /** + * File used to store the checksum for the instrumentation used by this plugin to create the instrumented Java + * installation + */ + private File checksumFile; + /** + * File used to store the path of the uninstrumented Java installation and fully qualified name of the + * implementation of {@link Instrumentation} used by this plugin to create the instrumented Java installation + */ + private File infoFile; + + /** + * Creates an instrumented Java installation. + * + * @throws MojoExecutionException if an instrumented Java installation could not be created + */ + @Override + public void execute() throws MojoExecutionException { + if (!InstrumentUtil.isJavaHome(javaHome)) { + throw new MojoExecutionException("Expected Java installation at: " + javaHome); + } + optionsFile = new File(outputDirectory, "phosphor-instrument" + File.separator + "option.properties"); + checksumFile = new File(optionsFile.getParent(), "checksum.md5"); + infoFile = new File(optionsFile.getParent(), "info.txt"); + Instrumentation instance = createInstrumentation(); + byte[] checksum = computeChecksum(instance); + String info = String.format("%s%n%s", javaHome.getAbsolutePath(), instrumentationType); + if (InstrumentUtil.isJavaHome(outputDirectory) + && checksumFile.isFile() + && optionsFile.isFile() + && infoFile.isFile()) { + if (!forceCreation && checkMatchFiles(checksum, info)) { + getLog().info("Existing instrumented Java installation with correct settings found: " + + outputDirectory); + getLog().info("Skipping creation."); + } else { + if (!forceCreation) { + getLog().info("Existing Java installation did not have correct settings."); + } + getLog().info("Recreating Java installation : " + outputDirectory); + deleteExistingJdkAndLinkedCaches(); + createInstrumentedJdk(instance, checksum, info); + } + } else if (outputDirectory.exists()) { + String message = "Failed to create instrumented Java installation." + + " %s already exists and is not an instrumented Java installation."; + throw new MojoExecutionException(String.format(message, outputDirectory)); + } else { + getLog().info("Creating Java installation : " + outputDirectory); + createInstrumentedJdk(instance, checksum, info); + } + } + + private Instrumentation createInstrumentation() throws MojoExecutionException { + try { + Class clazz = Class.forName(instrumentationType, true, getClass().getClassLoader()); + Instrumentation instance = (Instrumentation) clazz.getConstructor().newInstance(); + instance.configure(javaHome, outputDirectory, options); + return instance; + } catch (ClassCastException | IOException | ReflectiveOperationException e) { + throw new MojoExecutionException( + "Error occurred while creating instrumentation instance: " + instrumentationType, e); + } + } + + private boolean checkMatchFiles(byte[] checksum, String info) throws MojoExecutionException { + try (FileReader reader = new FileReader(optionsFile)) { + Properties foundOptions = new Properties(); + foundOptions.load(reader); + return foundOptions.equals(options) + && Arrays.equals(checksum, InstrumentUtil.readAllBytes(checksumFile)) + && new String(InstrumentUtil.readAllBytes(infoFile)).equals(info); + } catch (IOException e) { + throw new MojoExecutionException("Failed to read match info", e); + } + } + + private void deleteExistingJdkAndLinkedCaches() throws MojoExecutionException { + try { + Files.walkFileTree(outputDirectory.toPath(), new DeletingFileVisitor()); + for (File file : linkedCaches) { + if (file.exists()) { + Files.walkFileTree(file.toPath(), new DeletingFileVisitor()); + getLog().info("Deleted linked cache:" + file); + } + } + } catch (IOException e) { + throw new MojoExecutionException("Failed to delete existing instrumented Java installation and caches", e); + } + } + + private void createInstrumentedJdk(Instrumentation instance, byte[] checksum, String info) + throws MojoExecutionException { + try { + InstrumentDriver.instrument(javaHome, outputDirectory, instance); + } catch (IOException e) { + throw new MojoExecutionException("Failed to instrument Java instrumentation.", e); + } + writeMatchFiles(checksum, info); + } + + private void writeMatchFiles(byte[] checksum, String info) throws MojoExecutionException { + try { + InstrumentUtil.ensureDirectory(optionsFile.getParentFile()); + Files.write(checksumFile.toPath(), checksum); + Files.write(infoFile.toPath(), info.getBytes()); + try (FileWriter writer = new FileWriter(optionsFile)) { + options.store(writer, null); + } + } catch (IOException e) { + throw new MojoExecutionException("Failed to write match files", e); + } + } + + private static byte[] computeChecksum(Instrumentation instrumentation) throws MojoExecutionException { + try { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + for (File f : instrumentation.getClassPathElements()) { + if (f.isFile()) { + buffer.write(InstrumentUtil.readAllBytes(f)); + } + } + return InstrumentUtil.checksum(buffer.toByteArray()); + } catch (IOException e) { + throw new MojoExecutionException("Failed to compute instrumentation checksum", e); + } + } +} diff --git a/pom.xml b/pom.xml index 775790317..d0b6e72e4 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,20 @@ UTF-8 + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.4.1 + + + org.moditect + moditect-maven-plugin + 1.0.0.RC2 + + + maven-compiler-plugin @@ -67,7 +81,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.3.1 + 3.3.0 org.apache.maven.plugins @@ -90,7 +104,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.1.1 + 3.6.0 org.apache.maven.plugins @@ -130,6 +144,24 @@ org.codehaus.mojo 1.6.0 + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + 1.8 + none + true + + + + attach-javadoc + + jar + + + + From 7fa87272030e5d2a49d4164e9a0d24c3d25e6e92 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sat, 25 Nov 2023 16:05:24 -0500 Subject: [PATCH 02/12] * Continued working on new implementation of instrumenting plugin and driver --- Phosphor/pom.xml | 78 +-- .../cs/psl/phosphor/Instrumenter.java | 548 ++---------------- .../cs/psl/phosphor/Java8PreMain.java | 2 +- .../columbia/cs/psl/phosphor/OptionsUtil.java | 61 +- .../cs/psl/phosphor/PhosphorOption.java | 6 +- .../cs/psl/phosphor/PhosphorPatcher.java | 35 +- .../cs/psl/phosphor/InstrumenterTest.java | 14 +- README.md | 2 +- integration-tests/README.md | 22 +- integration-tests/pom.xml | 98 +++- integration-tests/runDacapo.sh | 2 +- phosphor-instrument-jigsaw/pom.xml | 10 +- .../phosphor/instrumenter/JLinkInvoker.java | 80 --- .../instrumenter/JLinkRegistrationAgent.java | 43 -- .../instrumenter/PhosphorJLinkPlugin.java | 108 ---- .../DeletingFileVisitor.java | 2 +- .../instrument/InstrumentJLinkPlugin.java | 70 +++ .../{jlink => instrument}/InstrumentUtil.java | 2 +- .../phosphor/instrument/Instrumentation.java | 46 ++ .../{jlink => instrument}/Instrumenter.java | 56 +- .../swe/phosphor/instrument/JLinkInvoker.java | 76 +++ .../JLinkRegistrationAgent.java | 13 +- .../swe/phosphor/instrument/Packer.java} | 93 +-- .../gmu/swe/phosphor/instrument/Patcher.java | 7 + .../instrument/PhosphorInstrumentation.java | 91 +++ .../instrument/PhosphorInstrumenter.java | 21 + .../swe/phosphor/jlink/InstrumentDriver.java | 54 -- .../phosphor/jlink/InstrumentJLinkPlugin.java | 128 ---- .../swe/phosphor/jlink/Instrumentation.java | 30 - .../gmu/swe/phosphor/jlink/JLinkInvoker.java | 68 --- .../jlink/PhosphorInstrumentation.java | 35 -- .../src/main/java/module-info.java | 4 +- ...rumentingMojo.java => InstrumentMojo.java} | 171 +++--- 33 files changed, 723 insertions(+), 1353 deletions(-) delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkRegistrationAgent.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorJLinkPlugin.java rename phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/{jlink => instrument}/DeletingFileVisitor.java (96%) create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java rename phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/{jlink => instrument}/InstrumentUtil.java (98%) create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java rename phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/{jlink => instrument}/Instrumenter.java (76%) create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java rename phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/{jlink => instrument}/JLinkRegistrationAgent.java (97%) rename phosphor-instrument-jigsaw/src/main/java/edu/{columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java => gmu/swe/phosphor/instrument/Packer.java} (52%) create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java create mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java rename phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/{InstrumentingMojo.java => InstrumentMojo.java} (53%) diff --git a/Phosphor/pom.xml b/Phosphor/pom.xml index b60f0540f..9d0ef5706 100644 --- a/Phosphor/pom.xml +++ b/Phosphor/pom.xml @@ -44,7 +44,6 @@ - org.codehaus.mojo exec-maven-plugin @@ -115,70 +114,17 @@ - org.apache.maven.pluginsmaven-compiler-plugin88 + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + - - - release-sign-artifacts - - - gpg.passphrase - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - - --pinentry-mode - loopback - - - - - - - - make-graph - - true - - - - - org.codehaus.mojo - exec-maven-plugin - - - test - - java - - - edu.columbia.cs.psl.phosphor.control.graph.BaseControlFlowGraphCreator - - {$exec.args} - - - - - - - - commons-cli @@ -210,11 +156,5 @@ asm-commons 9.6 - - commons-cli - commons-cli - 1.4 - compile - diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java index d513f81db..0308f3892 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Instrumenter.java @@ -1,37 +1,26 @@ package edu.columbia.cs.psl.phosphor; -import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; import edu.columbia.cs.psl.phosphor.runtime.StringUtils; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.*; -import org.apache.commons.cli.CommandLine; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.Collections; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.HashMap; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.Map; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.tree.ClassNode; import org.objectweb.asm.tree.MethodNode; -import java.io.*; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.channels.FileChannel; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Random; -import java.util.concurrent.*; -import java.util.zip.*; +import java.io.InputStream; import static edu.columbia.cs.psl.phosphor.Configuration.controlFlowManagerPackage; import static edu.columbia.cs.psl.phosphor.Configuration.taintTagFactoryPackage; -public class Instrumenter { +public final class Instrumenter { public static ClassLoader loader; public static Map classes = Collections.synchronizedMap(new HashMap<>()); public static InputStream sourcesFile; public static InputStream sinksFile; public static InputStream taintThroughFile; - static String curPath; - static int n = 0; static { classes.putAll(ClassSupertypeReadingTransformer.classNodes); @@ -39,7 +28,7 @@ public class Instrumenter { } private Instrumenter() { - // Prevents this class from being instantiated + throw new AssertionError("Tried to instantiate static utility class: " + getClass()); } public static boolean isIgnoredClass(String owner) { @@ -48,7 +37,7 @@ public static boolean isIgnoredClass(String owner) { || controlFlowManagerPackage != null && StringUtils.startsWith(owner, controlFlowManagerPackage) || (Configuration.controlFlowManager != null && Configuration.controlFlowManager.isIgnoredClass(owner)) || (Configuration.ADDL_IGNORE != null && StringUtils.startsWith(owner, Configuration.ADDL_IGNORE)) - //|| !owner.startsWith("edu/columbia/cs/psl") + // || !owner.startsWith("edu/columbia/cs/psl") // For these classes: HotSpot expects fields to be at hardcoded offsets of these classes. // If we instrument them, it will break those assumptions and segfault. // Different classes have different assumptions, and there are special cases for these elsewhere in @@ -62,9 +51,9 @@ public static boolean isIgnoredClass(String owner) { || StringUtils.startsWith(owner, "java/lang/ref/Reference") || StringUtils.startsWith(owner, "java/lang/ref/FinalReference") || StringUtils.startsWith(owner, "java/lang/ref/SoftReference") - //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool + // Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool || StringUtils.equals(owner, "java/lang/invoke/LambdaForm") - //Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool + // Lambdas are hosted by this class, and when generated, will have hard-coded offsets to constant pool || StringUtils.startsWith(owner, "java/lang/invoke/LambdaForm$") // Phosphor internal classes || StringUtils.startsWith(owner, "edu/columbia/cs/psl/phosphor") @@ -81,455 +70,22 @@ public static boolean isIgnoredClass(String owner) { || StringUtils.startsWith(owner, "jdk/internal/misc/UnsafeConstants"); } - public static byte[] instrumentClass(String path, InputStream is, boolean renameInterfaces) { - try { - // n is shared among threads, but is used only to provide progress feedback - // Therefore, it's ok to increment it in a non-thread-safe way - n++; - if(!Configuration.QUIET_MODE && n % 1000 == 0) { - System.out.println("Processed: " + n); - } - curPath = path; - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - byte[] data = new byte[16384]; - for(int nRead; (nRead = is.read(data, 0, data.length)) != -1;) { - buffer.write(data, 0, nRead); - } - is.close(); - buffer.flush(); - PreMain.PCLoggingTransformer transformer = new PreMain.PCLoggingTransformer(); - byte[] ret = transformer.transform(Instrumenter.loader, path, null, null, buffer.toByteArray(), false); - curPath = null; - return ret; - } catch(Exception e) { - curPath = null; - e.printStackTrace(); - throw new IllegalStateException(e); - } - } - - public static void main(String[] args) { - long START = System.currentTimeMillis(); - CommandLine line = PhosphorOption.configure(false, args); - if(line == null) { - return; - } - Configuration.init(); - if(Configuration.DATAFLOW_TRACKING) { - System.out.println("Data flow tracking: enabled"); - } else { - System.out.println("Data flow tracking: disabled"); - } - if(Configuration.IMPLICIT_TRACKING) { - System.out.println("Control flow tracking: enabled"); - } else { - System.out.println("Control flow tracking: disabled"); - } - if(Configuration.WITHOUT_BRANCH_NOT_TAKEN) { - System.out.println("Branch not taken: disabled"); - } else { - System.out.println("Branch not taken: enabled"); - } - TaintTrackingClassVisitor.IS_RUNTIME_INST = false; - _main(line.getArgs()); - System.out.println("Done after " + (System.currentTimeMillis() - START) + " ms"); - } - - public static void _main(String[] args) { - if(PreMain.DEBUG) { - System.err.println("Warning: Debug output enabled (uses a lot of IO!)"); - } - - String outputFolder = args[1]; - File rootOutputDir = new File(outputFolder); - if(!rootOutputDir.exists()) { - rootOutputDir.mkdir(); - } - String inputFolder = args[0]; - - // Setup the class loader - final ArrayList urls = new ArrayList<>(); - Path input = FileSystems.getDefault().getPath(args[0]); - try { - if(Files.isDirectory(input)) { - Files.walkFileTree(input, new FileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if(file.getFileName().toString().endsWith(".jar")) { - urls.add(file.toUri().toURL()); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - return FileVisitResult.CONTINUE; - } - }); - } else if(inputFolder.endsWith(".jar")) { - urls.add(new File(inputFolder).toURI().toURL()); - } - } catch(IOException e1) { - e1.printStackTrace(); - } - - try { - urls.add(new File(inputFolder).toURI().toURL()); - } catch(MalformedURLException e1) { - e1.printStackTrace(); - } - - - URL[] urlArray = new URL[urls.size()]; - urlArray = urls.toArray(urlArray); - loader = new URLClassLoader(urlArray, Instrumenter.class.getClassLoader()); - PreMain.bigLoader = loader; - - final File f = new File(inputFolder); - if(!f.exists()) { - System.err.println("Unable to read path " + inputFolder); - System.exit(-1); - } - - final ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - LinkedList toWait = new LinkedList<>(); - - if(f.isDirectory()) { - toWait.addAll(processDirectory(f, rootOutputDir, true, executor)); - } else if(inputFolder.endsWith(".jar") || inputFolder.endsWith(".zip") || inputFolder.endsWith(".war") - || inputFolder.endsWith(".jmod")) { - toWait.addAll(processZip(f, rootOutputDir, executor)); - } else if(inputFolder.endsWith(".class")) { - toWait.addAll(processClass(f, rootOutputDir, executor)); - } else { - System.err.println("Unknown type for path " + inputFolder); - System.exit(-1); - } - - while(!toWait.isEmpty()) { - try { - toWait.addAll((Collection) toWait.removeFirst().get()); - } catch(InterruptedException e) { - // - } catch(ExecutionException e) { - throw new Error(e); - } - } - - executor.shutdown(); - while(!executor.isTerminated()) { - try { - executor.awaitTermination(1, TimeUnit.SECONDS); - } catch(InterruptedException e) { - // - } - } - } - - private static List> processClass(File f, final File outputDir, ExecutorService executor) { - List> ret = new LinkedList<>(); - try { - final String name = f.getName(); - final InputStream is = new FileInputStream(f); - ret.add(executor.submit(new Callable() { - @Override - public List call() throws Exception { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - FileOutputStream fos = new FileOutputStream(outputDir.getPath() + File.separator + name); - byte[] c = instrumentClass(outputDir.getAbsolutePath(), is, true); - is.close(); - if(c != null) { - bos.write(c); - } - bos.writeTo(fos); - fos.close(); - return new LinkedList(); - } - })); - - } catch(Exception ex) { - ex.printStackTrace(); - } - return ret; - } - - private static LinkedList processDirectory(File f, File parentOutputDir, boolean isFirstLevel, ExecutorService executor) { - LinkedList ret = new LinkedList<>(); - if(f.getName().equals(".AppleDouble")) { - return ret; - } - final File thisOutputDir; - if(isFirstLevel) { - thisOutputDir = parentOutputDir; - } else { - thisOutputDir = new File(parentOutputDir.getAbsolutePath() + File.separator + f.getName()); - thisOutputDir.mkdir(); - } - for(final File fi : f.listFiles()) { - if(fi.isDirectory()) { - ret.addAll(processDirectory(fi, thisOutputDir, false, executor)); - } else if(fi.getName().endsWith(".class")) { - ret.addAll(processClass(fi, thisOutputDir, executor)); - } else if(fi.getName().endsWith(".jar") || fi.getName().endsWith(".zip") || fi.getName().endsWith(".war") - || fi.getName().endsWith(".jmod")) { - ret.addAll(processZip(fi, thisOutputDir, executor)); - } else { - File dest = new File(thisOutputDir.getPath() + File.separator + fi.getName()); - FileChannel source = null; - FileChannel destination = null; - - try { - source = new FileInputStream(fi).getChannel(); - destination = new FileOutputStream(dest).getChannel(); - destination.transferFrom(source, 0, source.size()); - if(fi.canExecute()) { - dest.setExecutable(true); - } - if(fi.canRead()) { - dest.setReadable(true); - } - if(fi.canWrite()) { - dest.setWritable(true); - } - } catch(Exception ex) { - System.err.println("error copying file " + fi); - ex.printStackTrace(); - } finally { - if(source != null) { - try { - source.close(); - } catch(IOException e) { - e.printStackTrace(); - } - } - if(destination != null) { - try { - destination.close(); - } catch(IOException e) { - e.printStackTrace(); - } - } - } - } - } - - return ret; - } - - /** - * Handles Jar files, Zip files and War files. - */ - public static LinkedList processZip(final File f, File outputDir, ExecutorService executor) { - return _processZip(f, outputDir, executor, false); - } - - private static LinkedList _processZip(final File f, File outputDir, ExecutorService executor, boolean unCompressed) { - try(ZipFile zip = new ZipFile(f); ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputDir.getPath() + File.separator + f.getName()))) { - if(unCompressed) { - zos.setLevel(ZipOutputStream.STORED); - } - LinkedList> ret = new LinkedList<>(); - java.util.Enumeration entries = zip.entries(); - while(entries.hasMoreElements()) { - final ZipEntry e = entries.nextElement(); - - if(e.getName().endsWith(".class")) { - ret.add(executor.submit(new Callable() { - @Override - public Result call() throws Exception { - Result ret = new Result(); - ret.e = e; - ret.buf = instrumentClass(f.getAbsolutePath(), zip.getInputStream(e), true); - return ret; - } - })); - } else if(e.getName().endsWith(".jar")) { - ZipEntry outEntry = new ZipEntry(e.getName()); - Random r = new Random(); - String markFileName = Long.toOctalString(System.currentTimeMillis()) - + Integer.toOctalString(r.nextInt(10000)) - + e.getName().replace("/", ""); - final String tempDir = System.getProperty("java.io.tmpdir"); - File tmp = new File(tempDir, markFileName); - if(tmp.exists()) { - tmp.delete(); - } - try(InputStream is = zip.getInputStream(e); FileOutputStream fos = new FileOutputStream(tmp)) { - byte[] buf = new byte[1024]; - for(int len; (len = is.read(buf)) > 0;) { - fos.write(buf, 0, len); - } - } - - File tmp2 = new File(tempDir, "tmp2"); - if(!tmp2.exists()) { - tmp2.mkdir(); - } - _processZip(tmp, tmp2, executor, true); - tmp.delete(); - - outEntry.setMethod(ZipEntry.STORED); - Path newPath = Paths.get(tempDir, "tmp2", markFileName); - - outEntry.setSize(Files.size(newPath)); - CRC32 crc = new CRC32(); - crc.update(Files.readAllBytes(newPath)); - outEntry.setCrc(crc.getValue()); - zos.putNextEntry(outEntry); - File newFile = newPath.toFile(); - try(InputStream is = new FileInputStream(newFile)) { - byte[] buffer = new byte[1024]; - for(int count = is.read(buffer); count != -1; count = is.read(buffer)) { - zos.write(buffer, 0, count); - } - } - newFile.delete(); - zos.closeEntry(); - } else { - ZipEntry outEntry = new ZipEntry(e.getName()); - if(e.isDirectory()) { - try { - zos.putNextEntry(outEntry); - zos.closeEntry(); - } catch(ZipException exxxx) { - System.out.println("Ignoring exception: " + exxxx.getMessage()); - } - } else if(!e.getName().startsWith("META-INF") - || (!e.getName().endsWith(".SF") - && !e.getName().endsWith(".RSA"))) { - if(e.getName().equals("META-INF/MANIFEST.MF")) { - Scanner s = new Scanner(zip.getInputStream(e)); - zos.putNextEntry(outEntry); - - String curPair = ""; - while(s.hasNextLine()) { - String line = s.nextLine(); - if(line.equals("")) { - curPair += "\n"; - if(!curPair.contains("SHA1-Digest:")) { - zos.write(curPair.getBytes()); - } - curPair = ""; - } else { - curPair += line + "\n"; - } - } - s.close(); - // Jar file is different from Zip file. :) - if(f.getName().endsWith(".zip")) { - zos.write("\n".getBytes()); - } - zos.closeEntry(); - } else { - try { - zos.putNextEntry(outEntry); - try(InputStream is = zip.getInputStream(e)) { - byte[] buffer = new byte[1024]; - for(int count = is.read(buffer); count != -1; count = is.read(buffer)) { - zos.write(buffer, 0, count); - } - } - zos.closeEntry(); - } catch(ZipException ex) { - if(!ex.getMessage().contains("duplicate entry")) { - ex.printStackTrace(); - System.out.println("Ignoring above warning from improper source zip..."); - } - } - } - } - } - } - for(Future fr : ret) { - Result r; - while(true) { - try { - r = fr.get(); - break; - } catch(InterruptedException e) { - // - } - } - try { - ZipEntry outEntry = new ZipEntry(r.e.getName()); - zos.putNextEntry(outEntry); - - byte[] clazz = r.buf; - if(clazz == null) { - System.out.println("Failed to instrument " + r.e.getName() + " in " + f.getName()); - try(InputStream is = zip.getInputStream(r.e)) { - byte[] buffer = new byte[1024]; - for(int count = is.read(buffer); count != -1; count = is.read(buffer)) { - zos.write(buffer, 0, count); - } - } - } else { - zos.write(clazz); - } - zos.closeEntry(); - } catch(ZipException ex) { - ex.printStackTrace(); - } - } - } catch(Exception e) { - System.err.println("Unable to process zip/jar: " + f.getAbsolutePath()); - e.printStackTrace(); - File dest = new File(outputDir.getPath() + File.separator + f.getName()); - FileChannel source = null; - FileChannel destination = null; - try { - source = new FileInputStream(f).getChannel(); - destination = new FileOutputStream(dest).getChannel(); - destination.transferFrom(source, 0, source.size()); - } catch(Exception ex) { - System.err.println("Unable to copy zip/jar: " + f.getAbsolutePath()); - ex.printStackTrace(); - } finally { - if(source != null) { - try { - source.close(); - } catch(IOException e2) { - e2.printStackTrace(); - } - } - if(destination != null) { - try { - destination.close(); - } catch(IOException e2) { - e2.printStackTrace(); - } - } - } - } - return new LinkedList<>(); - } - public static boolean isIgnoredMethod(String owner, String name, String desc) { - return false; //TODO see below from old jdk14 version - //if(name.equals("wait") && desc.equals("(J)V")) { + return false; // TODO see below from old jdk14 version + // if(name.equals("wait") && desc.equals("(J)V")) { // return true; - //} - //if(name.equals("wait") && desc.equals("(JI)V")) { + // } + // if(name.equals("wait") && desc.equals("(JI)V")) { // return true; - //} - //if (owner.equals("jdk/internal/reflect/Reflection") && name.equals("getCallerClass")) { + // } + // if (owner.equals("jdk/internal/reflect/Reflection") && name.equals("getCallerClass")) { // return true; - //} - //if (owner.equals("java/lang/invoke/MethodHandle") + // } + // if (owner.equals("java/lang/invoke/MethodHandle") // && ((name.equals("invoke") || name.equals("invokeBasic") || name.startsWith("linkTo")))) { // return true; - //} - //return owner.equals("java/lang/invoke/VarHandle"); //TODO wrap these all + // } + // return owner.equals("java/lang/invoke/VarHandle"); //TODO wrap these all } public static boolean isPolymorphicSignatureMethod(String owner, String name) { @@ -580,7 +136,7 @@ public static boolean isUninstrumentedField(String owner, String name) { * successfully be created for the class name. */ public static ClassNode getClassNode(String className) { ClassNode cn = classes.get(className); - if(cn == null) { + if (cn == null) { // Class was loaded before ClassSupertypeReadingTransformer was added return tryToAddClassNode(className); } else { @@ -590,49 +146,47 @@ public static ClassNode getClassNode(String className) { /* Attempts to create a ClassNode populated with supertype information for this class. */ private static ClassNode tryToAddClassNode(String className) { - try(InputStream is = ClassLoader.getSystemResourceAsStream(className + ".class")) { - if(is == null) { + try (InputStream is = ClassLoader.getSystemResourceAsStream(className + ".class")) { + if (is == null) { return null; } ClassReader cr = new ClassReader(is); - cr.accept(new ClassVisitor(Configuration.ASM_VERSION) { - private ClassNode cn; + cr.accept( + new ClassVisitor(Configuration.ASM_VERSION) { + private ClassNode cn; - @Override - public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { - super.visit(version, access, name, signature, superName, interfaces); - cn = new ClassNode(); - cn.name = name; - cn.superName = superName; - cn.interfaces = new java.util.ArrayList<>(java.util.Arrays.asList(interfaces)); - cn.methods = new java.util.LinkedList<>(); - classes.put(name, cn); - } + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + cn = new ClassNode(); + cn.name = name; + cn.superName = superName; + cn.interfaces = new java.util.ArrayList<>(java.util.Arrays.asList(interfaces)); + cn.methods = new java.util.LinkedList<>(); + classes.put(name, cn); + } - @Override - public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - cn.methods.add(new MethodNode(access, name, descriptor, signature, exceptions)); - return super.visitMethod(access, name, descriptor, signature, exceptions); - } - }, ClassReader.SKIP_CODE); + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + cn.methods.add(new MethodNode(access, name, descriptor, signature, exceptions)); + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + }, + ClassReader.SKIP_CODE); return classes.get(className); - } catch(Exception e) { + } catch (Exception e) { return null; } } - private static class Result { - ZipEntry e; - byte[] buf; - } - - public static boolean isJava8JVMDir(File java_home) { - return new File(java_home, "bin" + File.separator + "java").exists() - && !new File(java_home, "jmods").exists() - && !new File(java_home, "lib" + File.separator + "modules").exists(); - } - - public static boolean isUnsafeClass(String className){ + public static boolean isUnsafeClass(String className) { return (Configuration.IS_JAVA_8 && "sun/misc/Unsafe".equals(className)) || (!Configuration.IS_JAVA_8 && "jdk/internal/misc/Unsafe".equals(className)); } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java index 6f27e907a..bf8e02b8d 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java @@ -25,7 +25,7 @@ public PhosphorTransformerBridge(PhosphorBaseTransformer transformer) { @Override public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException { - return transform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer); + return transform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer, null); } public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer, PhosphorStackFrame phosphorStackFrame) throws IllegalClassFormatException { diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java index 4818d1a26..0bd356470 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java @@ -4,7 +4,6 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import java.io.File; import java.util.*; public class OptionsUtil { @@ -13,58 +12,46 @@ public class OptionsUtil { * to either true or not present in the properties, each Phosphor option with an argument is either mapped to a * non-null, non-empty string or not present in properties, and no other keys are present in the properties. * - * @param isRuntimeInst true if the options should be standardized against the options available during dynamic - * instrumentation, otherwise standardizes against the options available during static - * instrumentation - * @param properties un-standardized properties to be standardized + * @param properties properties to be standardized * @return a standardized copy of properties */ - public static Properties canonicalizeProperties(Properties properties, boolean isRuntimeInst) { - Set propNames = properties.stringPropertyNames(); - Properties canonicalProps = new Properties(); - Map phosphorOptionMap = createPhosphorOptionMap(isRuntimeInst); - for (String propName : propNames) { - if (phosphorOptionMap.containsKey(propName)) { - Option option = phosphorOptionMap.get(propName); - if (option.hasArg()) { - if (properties.getProperty(propName) != null - && !properties.getProperty(propName).isEmpty()) { - canonicalProps.setProperty(option.getOpt(), properties.getProperty(propName)); - } - } else { - if (properties.getProperty(propName).isEmpty() - || "true".equalsIgnoreCase(properties.getProperty(propName))) { - canonicalProps.setProperty(option.getOpt(), "true"); - } + private static Properties standardize(Properties properties) { + Properties result = new Properties(); + Map phosphorOptionMap = createOptionMap(); + for (String key : properties.stringPropertyNames()) { + String value = properties.getProperty(key); + if (phosphorOptionMap.containsKey(key)) { + Option option = phosphorOptionMap.get(key); + if (option.hasArg() && value != null && !value.isEmpty()) { + result.setProperty(option.getOpt(), value); + } else if (!option.hasArg() && (value == null || value.isEmpty() || "true".equalsIgnoreCase(value))) { + result.setProperty(option.getOpt(), "true"); } } else { - System.err.println("Unknown Phosphor option: " + propName); + throw new IllegalArgumentException("Unknown option: " + key); } } - return canonicalProps; + return result; } /** - * @param isRuntimeInst true if a map of options available during dynamic instrumentation should be returned, - * otherwise a map of option available during static instrumentation should be returned * @return a mapping from the names of configuration options available in Phosphor to an instance of * org.apache.commons.cli.Option that represents that configuration option */ - private static Map createPhosphorOptionMap(boolean isRuntimeInst) { - Map phosphorOptionMap = new HashMap<>(); - Options options = PhosphorOption.createOptions(isRuntimeInst); + private static Map createOptionMap() { + Map map = new HashMap<>(); + Options options = PhosphorOption.createOptions(false); for (Option option : options.getOptions()) { - phosphorOptionMap.put(option.getOpt(), option); + map.put(option.getOpt(), option); if (option.hasLongOpt()) { - phosphorOptionMap.put(option.getLongOpt(), option); + map.put(option.getLongOpt(), option); } } - return phosphorOptionMap; + return map; } - public static String[] createPhosphorMainArguments( - File inputDirectory, File outputDirectory, Properties properties) { - properties = canonicalizeProperties(properties, false); + public static String[] createPhosphorMainArguments(Properties properties) { + properties = standardize(properties); SinglyLinkedList arguments = new SinglyLinkedList<>(); Set propNames = properties.stringPropertyNames(); for (String propName : propNames) { @@ -73,8 +60,8 @@ public static String[] createPhosphorMainArguments( arguments.addLast(properties.getProperty(propName)); } } - arguments.addLast(inputDirectory.getAbsolutePath()); - arguments.addLast(outputDirectory.getAbsolutePath()); + arguments.addLast("temp/"); + arguments.addLast("temp2/"); return arguments.toArray(new String[0]); } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java index 8692f34d0..3087b1809 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java @@ -287,11 +287,7 @@ public void configure(boolean forRuntimeInst, boolean isPresent, CommandLine com JAVA_8(new PhosphorOptionBuilder(null, true, true).alternativeName("java8")) { @Override public void configure(boolean forRuntimeInst, boolean isPresent, CommandLine commandLine) { - if (isPresent) { - Configuration.IS_JAVA_8 = true; - } else { - Configuration.IS_JAVA_8 = false; - } + Configuration.IS_JAVA_8 = isPresent; } }, JVM_MODULES(new PhosphorOptionBuilder("For Java 9+ JVM generation: list of Java modules to include in instrumented JVM", diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorPatcher.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorPatcher.java index ea88626d3..b21dea36f 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorPatcher.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorPatcher.java @@ -4,23 +4,18 @@ import edu.columbia.cs.psl.phosphor.instrumenter.TaintMethodRecord; import edu.columbia.cs.psl.phosphor.runtime.jdk.unsupported.UnsafeProxy; import org.objectweb.asm.*; -import org.objectweb.asm.commons.ModuleHashesAttribute; -import org.objectweb.asm.commons.ModuleResolutionAttribute; -import org.objectweb.asm.commons.ModuleTargetAttribute; import org.objectweb.asm.tree.ClassNode; import org.objectweb.asm.tree.MethodNode; -import org.objectweb.asm.tree.ModuleExportNode; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Set; public class PhosphorPatcher { private final boolean patchUnsafeNames; - public PhosphorPatcher(InputStream unsafeContent) throws IOException { - this.patchUnsafeNames = shouldPatchUnsafeNames(unsafeContent); + public PhosphorPatcher(byte[] unsafeClassFileBuffer) { + this.patchUnsafeNames = shouldPatchUnsafeNames(unsafeClassFileBuffer); } public byte[] patch(String name, byte[] content) throws IOException { @@ -36,31 +31,9 @@ public byte[] patch(String name, byte[] content) throws IOException { } } - public byte[] transformBaseModuleInfo(InputStream in, Set packages) { - try { - ClassNode classNode = new ClassNode(); - ClassReader cr = new ClassReader(in); - Attribute[] attributes = new Attribute[]{new ModuleTargetAttribute(), - new ModuleResolutionAttribute(), - new ModuleHashesAttribute()}; - cr.accept(classNode, attributes, 0); - // Add exports - for (String packageName : packages) { - classNode.module.exports.add(new ModuleExportNode(packageName, 0, null)); - } - // Add packages - classNode.module.packages.addAll(packages); - ClassWriter cw = new ClassWriter(0); - classNode.accept(cw); - return cw.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static boolean shouldPatchUnsafeNames(InputStream in) throws IOException { + private static boolean shouldPatchUnsafeNames(byte[] in) { ClassNode cn = new ClassNode(); - new ClassReader(in).accept(cn, 0); + new ClassReader(in).accept(cn, ClassReader.SKIP_CODE); for (MethodNode mn : cn.methods) { if (mn.name.contains("putReference")) { return false; diff --git a/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java b/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java index 708b86879..97efc245f 100644 --- a/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java +++ b/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java @@ -8,9 +8,6 @@ import org.objectweb.asm.Label; import org.objectweb.asm.MethodVisitor; -import java.io.ByteArrayInputStream; -import java.io.InputStream; - import static org.objectweb.asm.Opcodes.*; public class InstrumenterTest { @@ -30,8 +27,7 @@ public void checkForError() { @Test public void testAggressivelyReduceMethodSizeGetStaticThenArrayLength() { ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - cw.visit(52, ACC_PUBLIC + ACC_SUPER, "Test", null, "java/lang/Object", - null); + cw.visit(52, ACC_PUBLIC + ACC_SUPER, "Test", null, "java/lang/Object", null); MethodVisitor mv; mv = cw.visitMethod(ACC_PUBLIC + ACC_STATIC, "example", "()V", null, null); mv.visitCode(); @@ -46,8 +42,8 @@ public void testAggressivelyReduceMethodSizeGetStaticThenArrayLength() { mv.visitMaxs(1, 0); mv.visitEnd(); cw.visitEnd(); - InputStream in = new ByteArrayInputStream(cw.toByteArray()); - Instrumenter.instrumentClass("Test", in, true); + byte[] classFileBuffer = cw.toByteArray(); + new PreMain.PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); } @Test @@ -90,7 +86,7 @@ public void wideDuplicates() { mv.visitInsn(RETURN); mv.visitMaxs(-1, -1); mv.visitEnd(); - InputStream in = new ByteArrayInputStream(cw.toByteArray()); - Instrumenter.instrumentClass("Test", in, true); + byte[] classFileBuffer = cw.toByteArray(); + new PreMain.PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); } } \ No newline at end of file diff --git a/README.md b/README.md index 9e6dd6a7e..e38893c6e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Phosphor is a system for performing dynamic taint analysis in the JVM, on commod Phosphor has been extensively developed since its original publication, and now includes many features and options not described in the OOPSLA 2014 paper. If you are looking to replicate our OOPSLA 2014 experiments, the easiest way to get the same version with certainty is to use this [VM Image with all relevant files here](http://academiccommons.columbia.edu/catalog/ac%3A182689), and to follow the [README with instructions for doing so here](https://www.dropbox.com/s/dmebj6k8izams6p/artifact-63-phosphor.pdf?dl=0). ### Refactoring Status and Roadmap -This branch contains what is nearly a complete rewrite of Phosphor, using a less fragile (but slower) approach to pass taint tags between methods. It also is the first version of Phosphor to support Java9+. +This branch contains what is nearly a complete rewrite of Phosphor, using a less fragile (but slower) approach to pass taint tags between methods. It also is the first version of Phosphor to support Java 9+. Remaining tasks for this branch before promotion: * Implement control tracking semantics (currently entirely unimplemented) diff --git a/integration-tests/README.md b/integration-tests/README.md index 9deace86e..42517de7f 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -1,4 +1,22 @@ ## Tips -To skip data flow instrumentation and tests add the option `-P-data-flow` to the Maven commands. -This will disable the `data-flow` profile. \ No newline at end of file +To skip a set of tests add the option `-P-` to Maven commands. +Where <TYPE> is one of "data-flow" or "instrument". +This option will disable the profile which is used to run tests of the specified type. +This is useful if you wish to rerun and debug a single test class. +For example, if you wanted to run and debug only `ArrayLengthObjTagITCase`, you would run: + +```bash +mvn -P-instrument -Dmaven.failsafe.debug -Dit.test=ArrayLengthObjTagITCase verify +``` + +If an instrumented Java installation has already been created, and you want to run a minimal amount of plugins when +running a test you have to invoke the maven-dependency-plugin before failsafe. +For example, +```bash +mvn -P-instrument -Dit.test=ArrayLengthObjTagITCase dependency:properties failsafe:integration-test@data-flow +``` + +Add the option `-Dphosphor.forceCreation` to Maven commands to force the phosphor-instrument-maven-plugin to create a +new instrumented Java installation. +This will also delete associated caches of dynamically instrumented classes. \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index d2faebcfb..10beb587f 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -14,9 +14,12 @@ taintSources=${test.resources.dir}/taint-sources,taintSinks=${test.resources.dir}/taint-sinks,taintThrough=${test.resources.dir}/taint-through - ${project.build.directory}/phosphor/data/jdk/ + ${project.build.directory}/phosphor/data/java/ ${project.build.directory}/phosphor/data/cache false + ${edu.gmu.swe.phosphor:Phosphor:jar} + ${edu.gmu.swe.phosphor:phosphor-jigsaw-javaagent:jar} + false @@ -32,6 +35,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + test-compile + + testCompile + + + + maven-dependency-plugin @@ -45,13 +60,21 @@ org.apache.maven.plugins - maven-compiler-plugin + maven-enforcer-plugin - test-compile + enforce + validate - testCompile + enforce + + + + [8,) + + + @@ -67,7 +90,7 @@ none - ${data.flow.jdk} + ${data.flow.java} ${data.flow.cache} @@ -95,16 +118,6 @@ ${edu.gmu.swe.phosphor:Phosphor:jar} - - java9Plus - - [9,) - - - false - ${edu.gmu.swe.phosphor:phosphor-jigsaw-javaagent:jar} - - dacapo @@ -133,9 +146,9 @@ ${basedir}/runDacapo.sh ${project.build.directory} - ${edu.gmu.swe.phosphor:Phosphor:jar} + ${phosphor.jar} ${phosphor.agent} - ${data.flow.jdk}/bin/java + ${data.flow.java}/bin/java ${isJava8} @@ -170,23 +183,68 @@ data-flow + + integration-test + verify + - ${data.flow.jdk}/bin/java + ${data.flow.java}/bin/java **/*ObjTagITCase.java - **/*InstCase.java false -DphosphorCacheDirectory=${data.flow.cache} - -Xbootclasspath/a:${edu.gmu.swe.phosphor:Phosphor:jar} + -Xbootclasspath/a:${phosphor.jar} -javaagent:${phosphor.agent}=${auto.taint},enum,acmpeq + + + + + + + + instrument + + + !skipTests + + + + + + edu.gmu.swe.phosphor + phosphor-instrument-maven-plugin + + + data-flow + process-test-resources + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + instrument integration-test verify + + ${data.flow.java}/bin/java + + **/*InstCase.java + + false + + -Xbootclasspath/a:${phosphor.jar} + -javaagent:${phosphor.agent}=${auto.taint},enum,acmpeq + + diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index cf7eef842..a82206ff8 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -36,7 +36,7 @@ for bm in "${BENCHMARKS[@]}"; do # echo "$JAVA_HOME/bin/java -cp $DACAPO_DIR Harness $bm" if [ "$IS_JAVA_8" == "true" ]; then - $INST_JVM -Xmx1g -Xbootclasspath/p:"$PHOSPHOR_JAR" -javaagent:"$PHOSPHOR_JAVA_AGENT" -cp "$INST_DACAPO_DIR" -Declipse.java.home="$JAVA_HOME" -Djava.awt.headless=true Harness "$bm" + $INST_JVM -Xmx1g -Xbootclasspath/a:"$PHOSPHOR_JAR" -javaagent:"$PHOSPHOR_JAVA_AGENT" -cp "$INST_DACAPO_DIR" -Declipse.java.home="$JAVA_HOME" -Djava.awt.headless=true Harness "$bm" else $INST_JVM -Xmx4g -javaagent:"$PHOSPHOR_JAVA_AGENT" -cp "$INST_DACAPO_DIR" -Declipse.java.home="$JAVA_HOME" -Djava.awt.headless=true Harness "$bm" fi diff --git a/phosphor-instrument-jigsaw/pom.xml b/phosphor-instrument-jigsaw/pom.xml index 3f62873ba..e7720654c 100644 --- a/phosphor-instrument-jigsaw/pom.xml +++ b/phosphor-instrument-jigsaw/pom.xml @@ -59,9 +59,11 @@ - edu.gmu.swe.phosphor.jlink.JLinkRegistrationAgent + edu.gmu.swe.phosphor.instrument.JLinkRegistrationAgent - edu.gmu.swe.phosphor.jlink.InstrumentDriver + + edu.gmu.swe.phosphor.instrument.PhosphorInstrumenter + true @@ -81,13 +83,13 @@ org.jacoco.core - edu.gmu.swe.phosphor.jlink.org.jacoco.core + edu.gmu.swe.phosphor.instrument.org.jacoco.core org.objectweb.asm - edu.gmu.swe.phosphor.jlink.org.objectweb.asm + edu.gmu.swe.phosphor.instrument.org.objectweb.asm diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java deleted file mode 100644 index 35b63f5a6..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkInvoker.java +++ /dev/null @@ -1,80 +0,0 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - -import edu.columbia.cs.psl.phosphor.OptionsUtil; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; - -public class JLinkInvoker { - public static final String MODULES_PROPERTY = "jvmModules"; - public static final String PACK_KEY = "phosphor.pack"; - - public static void invokeJLink(File jvmDir, File instJVMDir, Properties properties) { - String jlinkBin = jvmDir + File.separator + "bin" + File.separator + "jlink"; - File jlinkFile = getClassPathElement(JLinkInvoker.class); - String modulesToAdd = - properties.getProperty(MODULES_PROPERTY, "java.base,jdk.jdwp.agent,java.instrument,jdk.unsupported"); - String classPath = buildClassPath(properties); - ProcessBuilder pb = new ProcessBuilder( - jlinkBin, - String.format("-J-D%s=%s", PACK_KEY, classPath), - "-J-javaagent:" + jlinkFile, - "-J--module-path=" + jlinkFile, - "-J--add-modules=edu.columbia.cs.psl.jigsaw.phosphor.instrumenter", - "-J--class-path=" + classPath, - "--output=" + instJVMDir, - "--phosphor-transformer=transform" + createPhosphorJLinkPluginArgument(properties), - "--add-modules=" + modulesToAdd); - try { - System.out.println(String.join(" ", pb.command())); - Process p = pb.inheritIO().start(); - p.waitFor(); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - - private static String buildClassPath(Properties properties) { - return OptionsUtil.getConfigurationClasses(properties).stream() - .map(JLinkInvoker::getClassPathElement) - .map(File::getAbsolutePath) - .collect(Collectors.joining(File.pathSeparator)); - } - - /** - * @param properties canonicalized properties that specify the Phosphor configuration options that should set in the - * created argument - * @return a String formatted for {@link PhosphorJLinkPlugin}'s arguments - * String argument - */ - public static String createPhosphorJLinkPluginArgument(Properties properties) { - if (properties.isEmpty()) { - return ""; - } else { - StringBuilder builder = new StringBuilder(); - Set propNames = properties.stringPropertyNames(); - for (String propName : propNames) { - if (propName.equals(MODULES_PROPERTY)) { - continue; - } - builder.append(':'); - builder.append(propName); - builder.append('=').append(properties.getProperty(propName)); - } - return builder.toString(); - } - } - - public static File getClassPathElement(Class clazz) { - try { - return new File( - clazz.getProtectionDomain().getCodeSource().getLocation().toURI()); - } catch (URISyntaxException e) { - throw new AssertionError(); - } - } -} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkRegistrationAgent.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkRegistrationAgent.java deleted file mode 100644 index 31dd528c7..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/JLinkRegistrationAgent.java +++ /dev/null @@ -1,43 +0,0 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - -import java.lang.instrument.Instrumentation; -import java.util.*; - - -/** - * Based heavily on hibernate-demos JLinkPluginRegistrationAgent, licensed under Apache License Version 2.0 - * https://github.com/hibernate/hibernate-demos/blob/master/java9/custom-jlink-plugin/agent/src/main/java/org/hibernate/demos/jlink/agent/JLinkPluginRegistrationAgent.java - */ -public class JLinkRegistrationAgent { - public static void premain(String agentArgs, Instrumentation inst) throws Exception { - Module jlinkModule = ModuleLayer.boot().findModule("jdk.jlink").get(); - Module phosphorModule = ModuleLayer.boot().findModule("edu.columbia.cs.psl.jigsaw.phosphor.instrumenter").get(); - Map> extraExports = new HashMap<>(); - extraExports.put("jdk.tools.jlink.plugin", Collections.singleton(phosphorModule)); - - // alter jdk.jlink to export its API to the module with our indexing plug-in - inst.redefineModule(jlinkModule, - Collections.emptySet(), - extraExports, - Collections.emptyMap(), - Collections.emptySet(), - Collections.emptyMap() - ); - - Class pluginClass = jlinkModule.getClassLoader().loadClass("jdk.tools.jlink.plugin.Plugin"); - Class addPhosphorPluginClass = phosphorModule.getClassLoader().loadClass("edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.PhosphorJLinkPlugin"); - - Map, List>> extraProvides = new HashMap<>(); - extraProvides.put(pluginClass, Collections.singletonList(addPhosphorPluginClass)); - - // alter the module with the phosphor plug-in so it provides the plug-in as a service - inst.redefineModule(phosphorModule, - Collections.emptySet(), - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptySet(), - extraProvides - ); - - } -} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorJLinkPlugin.java b/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorJLinkPlugin.java deleted file mode 100644 index cd0f75be6..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorJLinkPlugin.java +++ /dev/null @@ -1,108 +0,0 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; - -import edu.columbia.cs.psl.phosphor.*; -import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; -import jdk.tools.jlink.plugin.Plugin; -import jdk.tools.jlink.plugin.ResourcePool; -import jdk.tools.jlink.plugin.ResourcePoolBuilder; -import jdk.tools.jlink.plugin.ResourcePoolEntry; - -import java.io.File; -import java.util.*; - - -public class PhosphorJLinkPlugin implements Plugin { - public static final String NAME = "phosphor-transformer"; - private Set elementsToPack; - - @Override - public boolean hasArguments() { - return true; - } - - @Override - public Category getType() { - return Category.ADDER; - } - - /** - * @param properties canonicalized properties that specify the Phosphor configuration options that should set in the - * created arguments - * @return an array formatted for {@link Instrumenter#main(String[])} Instrumenter.main's} String[] argument - */ - public static String[] createPhosphorMainArguments(Map properties) { - LinkedList arguments = new LinkedList<>(); - Set propNames = properties.keySet(); - for (String propName : propNames) { - if (propName.equals("phosphor-transformer")) { - continue; - } - arguments.addLast("-" + propName); - if (!"true".equals(properties.get(propName))) { - arguments.addLast(properties.get(propName)); - } - } - arguments.addLast("ignored");//for input dir - arguments.addLast("ignored");//for output dir - return arguments.toArray(new String[0]); - } - - @Override - public void configure(Map config) { - Configuration.IS_JAVA_8 = false; - TaintTrackingClassVisitor.IS_RUNTIME_INST = false; - TaintUtils.VERIFY_CLASS_GENERATION = true; - PhosphorOption.configure(false, createPhosphorMainArguments(config)); - File phosphorJar = JLinkInvoker.getClassPathElement(Instrumenter.class); - System.out.println("Embedding Phosphor from: " + phosphorJar); - String pack = System.getProperty(JLinkInvoker.PACK_KEY); - this.elementsToPack = new HashSet<>(); - elementsToPack.add(phosphorJar); - if (pack != null && !pack.isEmpty()) { - Arrays.stream(pack.split(File.pathSeparator)) - .map(File::new) - .forEach(elementsToPack::add); - } - //TODO process args - } - - @Override - public ResourcePool transform(ResourcePool in, ResourcePoolBuilder out) { - TaintUtils.VERIFY_CLASS_GENERATION = true; - PreMain.RUNTIME_INST = false; - PhosphorPacker packer = new PhosphorPacker(in, elementsToPack); - in.transformAndCopy((resourcePoolEntry) -> { - if (resourcePoolEntry.type().equals(ResourcePoolEntry.Type.CLASS_OR_RESOURCE)) { - if (resourcePoolEntry.path().endsWith(".class")) { - if (resourcePoolEntry.path().endsWith("module-info.class")) { - //if this is the module for java-base, hack it to export phosphor, and pack phosphor into the module - if (resourcePoolEntry.path().startsWith("/java.base")) { - //This is the java.base module-info.class file. Transform it, and then add in phosphor - resourcePoolEntry = packer.pack(resourcePoolEntry, out); - } - } else { - byte[] newContent = Instrumenter.instrumentClass(resourcePoolEntry.path(), - resourcePoolEntry.content(), true); - if (newContent != null) { - resourcePoolEntry = resourcePoolEntry.copyWithContent(newContent); - } - } - } - } - return resourcePoolEntry; - }, out); - return out.build(); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getDescription() { - return "Transforms the runtime image to be compatible with Phosphor"; - } -} - - diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java similarity index 96% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java index 53894db7f..3fef97a16 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/DeletingFileVisitor.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.jlink; +package edu.gmu.swe.phosphor.instrument; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java new file mode 100644 index 000000000..53debf655 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java @@ -0,0 +1,70 @@ +package edu.gmu.swe.phosphor.instrument; + +import jdk.tools.jlink.plugin.Plugin; +import jdk.tools.jlink.plugin.ResourcePool; +import jdk.tools.jlink.plugin.ResourcePoolBuilder; +import jdk.tools.jlink.plugin.ResourcePoolEntry; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +public class InstrumentJLinkPlugin implements Plugin { + private Instrumentation instrumentation; + private Packer packer; + + @Override + public String getName() { + return "phosphor-instrument"; + } + + @Override + public Category getType() { + return Category.FILTER; + } + + @Override + public String getDescription() { + return "Applies instrumentation to the runtime image and packs classes into the java.base module"; + } + + @Override + public boolean hasArguments() { + return true; + } + + @Override + public void configure(Map config) { + try (FileReader reader = new FileReader(config.get("options"))) { + Properties options = new Properties(); + options.load(reader); + instrumentation = Instrumentation.create(config.get("type"), new File(config.get("source")), options); + } catch (IOException | ReflectiveOperationException e) { + throw new RuntimeException("Failed to process configuration", e); + } + } + + @Override + public ResourcePool transform(ResourcePool in, ResourcePoolBuilder out) { + packer = new Packer(in, instrumentation); + in.transformAndCopy(e -> transform(e, out), out); + return out.build(); + } + + private ResourcePoolEntry transform(ResourcePoolEntry entry, ResourcePoolBuilder out) { + if (entry.type().equals(ResourcePoolEntry.Type.CLASS_OR_RESOURCE) && entry.path().endsWith(".class")) { + if (entry.path().endsWith("module-info.class")) { + if (entry.path().startsWith("/java.base")) { + // Transform java.base's module-info.class file and pack core classes into java.base + return packer.pack(entry, out); + } + } else { + byte[] instrumented = instrumentation.apply(entry.contentBytes()); + return instrumented == null ? entry : entry.copyWithContent(instrumented); + } + } + return entry; + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java similarity index 98% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java index 53b3aae6d..0c6fb2b1e 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentUtil.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.jlink; +package edu.gmu.swe.phosphor.instrument; import java.io.*; import java.security.MessageDigest; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java new file mode 100644 index 000000000..870ab558b --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java @@ -0,0 +1,46 @@ +package edu.gmu.swe.phosphor.instrument; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.Set; +import java.util.function.Function; + +/** + * Implementors are expected to have a zero-argument, public constructor. + */ +public interface Instrumentation { + /** + * Configures this instance using the specified options. + * + * @param source the location to be instrumented + * @param options the key-value pairs that should be used to configure this instance + * @throws IOException if an I/O error occurs + */ + void configure(File source, Properties options) throws IOException, ReflectiveOperationException; + + /** + * Returns an array of class path elements needed to use this class. + * The returned array should be non-null. + * All elements of the returned array should be non-null. + * + * @return class path elements needed to use this class + */ + Set getClassPathElements(); + + byte[] apply(byte[] classFileBuffer); + + boolean shouldPack(String classFileName); + + Set getElementsToPack(); + + Patcher createPatcher(Function entryLocator); + + static Instrumentation create(String className, File source, Properties options) + throws ReflectiveOperationException, IOException { + Class clazz = Class.forName(className, true, Instrumentation.class.getClassLoader()); + Instrumentation instance = (Instrumentation) clazz.getConstructor().newInstance(); + instance.configure(source, options); + return instance; + } +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java similarity index 76% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java index 21eec919f..87e645525 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumenter.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.jlink; +package edu.gmu.swe.phosphor.instrument; import org.jacoco.core.internal.InputStreams; import org.jacoco.core.internal.instr.SignatureRemover; @@ -7,6 +7,7 @@ import java.nio.file.Files; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.*; public final class Instrumenter { @@ -15,12 +16,15 @@ public final class Instrumenter { Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); private final ConcurrentLinkedQueue errors = new ConcurrentLinkedQueue<>(); private final Instrumentation instrumentation; + private final AtomicInteger count = new AtomicInteger(0); + private final boolean verbose; - public Instrumenter(Instrumentation instrumentation) { + public Instrumenter(Instrumentation instrumentation, boolean verbose) { if (instrumentation == null) { throw new NullPointerException(); } this.instrumentation = instrumentation; + this.verbose = verbose; } public void process(File source, File target) throws IOException, InterruptedException, ExecutionException { @@ -50,16 +54,19 @@ public void process(File source, File target) throws IOException, InterruptedExc private void instrumentClass(File source, File target) { try (InputStream input = Files.newInputStream(source.toPath()); OutputStream output = Files.newOutputStream(target.toPath())) { - instrumentClass(input, output); + instrumentClass(InstrumentUtil.readAllBytes(input), output); } catch (Throwable t) { errors.add(t); } } - private void instrumentClass(InputStream input, OutputStream output) throws IOException { - byte[] buffer = InstrumentUtil.readAllBytes(input); - byte[] result = instrumentation.apply(buffer); - output.write(result == null ? buffer : result); + private void instrumentClass(byte[] classFileBuffer, OutputStream output) throws IOException { + byte[] result = instrumentation.apply(classFileBuffer); + output.write(result == null ? classFileBuffer : result); + int n; + if ((n = count.incrementAndGet()) % 1000 == 0 && verbose) { + System.out.println("Processed: " + n); + } } private void processFile(Collection> futures, File source, File target) @@ -76,13 +83,13 @@ private void processFile(Collection> futures, File source, File tar } else { if (copy(source, target)) { if (source.canExecute() && !target.setExecutable(true)) { - errors.add(new IOException("Failed to set permissions for: " + target)); + errors.add(new IOException("Failed to set execute permission for: " + target)); } if (source.canRead() && !target.setReadable(true)) { - errors.add(new IOException("Failed to set permissions for: " + target)); + errors.add(new IOException("Failed to set read permission for: " + target)); } if (source.canWrite() && !target.setWritable(true)) { - errors.add(new IOException("Failed to set permissions for: " + target)); + errors.add(new IOException("Failed to set write permission for: " + target)); } } } @@ -169,7 +176,7 @@ public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedE ByteArrayInputStream in = new ByteArrayInputStream(buffer); ByteArrayOutputStream out = new ByteArrayOutputStream(); if (entry.getName().endsWith(".class")) { - instrumentClass(in, out); + instrumentClass(buffer, out); } else if (entry.getName().endsWith(".jar")) { processZip(in, out); } else if (!signatureRemover.filterEntry(entry.getName(), in, out)) { @@ -183,4 +190,31 @@ public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedE this.buffer = tempBuffer; } } + + public static long instrument( + File source, + File target, + Properties options, + Instrumentation instrumentation, + boolean verbose, + String modules) + throws IOException { + if (target.exists()) { + throw new IllegalArgumentException("Target location for instrumentation already exists."); + } + if (!source.exists()) { + throw new IllegalArgumentException("Source location not found: " + source); + } + long startTime = System.currentTimeMillis(); + try { + if (InstrumentUtil.isModularJvm(source)) { + JLinkInvoker.invoke(source, target, instrumentation, options, modules); + } else { + new Instrumenter(instrumentation, verbose).process(source, target); + } + return System.currentTimeMillis() - startTime; + } catch (IOException | InterruptedException | ExecutionException e) { + throw new IOException("Failed to instrument source location", e); + } + } } \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java new file mode 100644 index 000000000..c67a485c4 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java @@ -0,0 +1,76 @@ +package edu.gmu.swe.phosphor.instrument; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.*; +import java.util.stream.Collectors; + +public final class JLinkInvoker { + private JLinkInvoker() { + throw new AssertionError(); + } + + public static void invoke( + File javaHome, File outputDirectory, Instrumentation instrumentation, Properties options, String modules) + throws InterruptedException, IOException { + String jlinkAgentJar = + InstrumentUtil.getClassPathElement(JLinkRegistrationAgent.class).getAbsolutePath(); + List command = new ArrayList<>(); + String classPath = buildClassPath(instrumentation); + command.add(InstrumentUtil.javaHomeToJLinkExec(javaHome).getAbsolutePath()); + command.add("-J-javaagent:" + jlinkAgentJar); + command.add("-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005"); + command.add("-J--class-path=" + classPath); + command.add("-J--add-reads=" + JLinkRegistrationAgent.MODULE_NAME + "=ALL-UNNAMED"); + command.add("-J--module-path=" + jlinkAgentJar); + command.add("-J--add-modules=" + JLinkRegistrationAgent.MODULE_NAME); + command.add(getPluginOption(javaHome, instrumentation, options)); + command.add("--output=" + outputDirectory.getAbsolutePath()); + command.add("--add-modules"); + command.add(processModules(modules)); + ProcessBuilder builder = new ProcessBuilder(command); + System.out.println(String.join(" ", builder.command())); + Process process = builder.inheritIO().start(); + if (process.waitFor() != 0) { + throw new RuntimeException("Failed to create instrumented runtime image"); + } + } + + private static File storeOptions(Properties options) throws IOException { + // Write the options to a temporary file + File file = Files.createTempFile("phosphor-", ".properties").toFile(); + file.deleteOnExit(); + InstrumentUtil.ensureDirectory(file.getParentFile()); + try (FileWriter writer = new FileWriter(file)) { + options.store(writer, null); + } + return file; + } + + private static String buildClassPath(Instrumentation instrumentation) { + return instrumentation.getClassPathElements().stream() + .map(File::getAbsolutePath) + .collect(Collectors.joining(File.pathSeparator)); + } + + private static String getPluginOption(File javaHome, Instrumentation instrumentation, Properties options) + throws IOException { + return String.format( + "--phosphor-instrument=x:type=%s:source=%s:options=%s", + instrumentation.getClass().getName(), + javaHome.getAbsolutePath(), + storeOptions(options).getAbsolutePath()); + } + + private static String processModules(String moduleString) { + // Ensure that all required modules are included + Set modules = + new LinkedHashSet<>(Arrays.asList("java.base", "jdk.jdwp.agent", "java.instrument", "jdk.unsupported")); + if (!moduleString.isEmpty()) { + modules.addAll(Arrays.asList(moduleString.split(","))); + } + return String.join(",", modules); + } +} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java similarity index 97% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java index 845b3f59f..3512d4ce7 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkRegistrationAgent.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java @@ -204,7 +204,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package edu.gmu.swe.phosphor.jlink; +package edu.gmu.swe.phosphor.instrument; import java.lang.instrument.Instrumentation; import java.util.*; @@ -213,7 +213,7 @@ * Alters the jdk.jlink module to export the "jdk.tools.jlink.plugin" package to this class' module. * Alters this class' module to provide our custom JLink plugin as a service. */ -@SuppressWarnings("Java9ReflectionClassVisibility") +@SuppressWarnings("Since15") public final class JLinkRegistrationAgent { public static final String MODULE_NAME = "edu.gmu.swe.phosphor.instrument"; private static final String JLINK_MODULE_NAME = "jdk.jlink"; @@ -222,12 +222,12 @@ public final class JLinkRegistrationAgent { public static void premain(String agentArgs, Instrumentation inst) throws Exception { Module jlinkModule = ModuleLayer.boot().findModule(JLINK_MODULE_NAME).orElseThrow(RuntimeException::new); - Module customPluginModule = - ModuleLayer.boot().findModule(MODULE_NAME).orElseThrow(RuntimeException::new); + Module customPluginModule = ModuleLayer.boot().findModule(MODULE_NAME).orElseThrow(RuntimeException::new); Map> extraExports = new HashMap<>(); extraExports.put(JLINK_PLUGIN_PACKAGE_NAME, Collections.singleton(customPluginModule)); // Alter jdk.jlink to export the Plugin API to the module containing our custom JLink plugin - inst.redefineModule(jlinkModule, + inst.redefineModule( + jlinkModule, Collections.emptySet(), extraExports, Collections.emptyMap(), @@ -237,7 +237,8 @@ public static void premain(String agentArgs, Instrumentation inst) throws Except Map, List>> extraProvides = new HashMap<>(); extraProvides.put(jlinkPluginClass, Collections.singletonList(InstrumentJLinkPlugin.class)); // Alter our custom JLink plugin's module to provide our custom JLink plugin as a service - inst.redefineModule(customPluginModule, + inst.redefineModule( + customPluginModule, Collections.emptySet(), Collections.emptyMap(), Collections.emptyMap(), diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java similarity index 52% rename from phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java rename to phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java index 4e2e8b77f..ec124b62c 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/instrumenter/PhosphorPacker.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java @@ -1,53 +1,46 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.instrumenter; +package edu.gmu.swe.phosphor.instrument; -import edu.columbia.cs.psl.phosphor.PhosphorPatcher; -import edu.gmu.swe.phosphor.jlink.InstrumentUtil; import jdk.tools.jlink.plugin.ResourcePool; import jdk.tools.jlink.plugin.ResourcePoolBuilder; import jdk.tools.jlink.plugin.ResourcePoolEntry; +import org.objectweb.asm.Attribute; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.commons.ModuleHashesAttribute; +import org.objectweb.asm.commons.ModuleResolutionAttribute; +import org.objectweb.asm.commons.ModuleTargetAttribute; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.ModuleExportNode; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -class PhosphorPacker { - private final PhosphorPatcher patcher; - private final Set elementsToPack; +public class Packer { + private final Instrumentation instrumentation; + private final Patcher patcher; - PhosphorPacker(ResourcePool pool, Set elementsToPack) { - this.elementsToPack = Collections.unmodifiableSet(new HashSet<>(elementsToPack)); - ResourcePoolEntry entry = pool.findEntry("/java.base/jdk/internal/misc/Unsafe.class") - .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for jdk/internal/misc/Unsafe")); - try (InputStream in = entry.content()) { - this.patcher = new PhosphorPatcher(in); - } catch (IOException e) { - throw new RuntimeException("Unable to read entry for jdk/internal/misc/Unsafe", e); + public Packer(ResourcePool in, Instrumentation instrumentation) { + if (in == null || instrumentation == null) { + throw new NullPointerException(); } + this.instrumentation = instrumentation; + this.patcher = instrumentation.createPatcher(path -> Packer.findEntry(in, path)); } - private boolean shouldInclude(String name) { - return !name.startsWith("edu/columbia/cs/psl/jigsaw/phosphor/instrumenter") - && !name.endsWith("module-info.class") - && !name.startsWith("org/") - && !name.startsWith("edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported") - && name.endsWith(".class"); - } - - ResourcePoolEntry pack(ResourcePoolEntry entry, ResourcePoolBuilder out) { - // Transform java.base's module-info.class file and pack Phosphor classes into java.base + public ResourcePoolEntry pack(ResourcePoolEntry entry, ResourcePoolBuilder out) { try { + // Pack classes into the java.bas Set packages = packClasses(out); + // Transform java.base's module-info.class file try (InputStream in = entry.content()) { - return entry.copyWithContent(patcher.transformBaseModuleInfo(in, packages)); + return entry.copyWithContent(transformBaseModuleInfo(in, packages)); } } catch (IOException e) { throw new RuntimeException(e); @@ -58,7 +51,7 @@ private Set packClasses(ResourcePoolBuilder out) throws IOException { // Pack the JARs and directories into the resource pool // Return the set of packages for packed classes Set packages = new HashSet<>(); - for (File element : elementsToPack) { + for (File element : instrumentation.getElementsToPack()) { if (element.isDirectory()) { packDirectory(out, element, packages); } else { @@ -70,7 +63,7 @@ private Set packClasses(ResourcePoolBuilder out) throws IOException { private void packClass(ResourcePoolBuilder out, String name, File classFile, Set packages) { try { - if (shouldInclude(name)) { + if (instrumentation.shouldPack(name)) { byte[] content = patcher.patch(name, Files.readAllBytes(classFile.toPath())); out.add(ResourcePoolEntry.create("/java.base/" + name, content)); packages.add(name.substring(0, name.lastIndexOf('/'))); @@ -83,7 +76,9 @@ private void packClass(ResourcePoolBuilder out, String name, File classFile, Set private void packDirectory(ResourcePoolBuilder out, File directory, Set packages) throws IOException { try (Stream walk = Files.walk(directory.toPath())) { walk.filter(Files::isRegularFile) - .forEach(p -> packClass(out, directory.toPath().relativize(p).toFile().getPath(), + .forEach(p -> packClass( + out, + directory.toPath().relativize(p).toFile().getPath(), p.toAbsolutePath().toFile(), packages)); } @@ -94,7 +89,7 @@ private void packJar(ResourcePoolBuilder out, File element, Set packages Enumeration entries = zip.entries(); while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); - if (shouldInclude(entry.getName())) { + if (instrumentation.shouldPack(entry.getName())) { try (InputStream is = zip.getInputStream(entry)) { byte[] content = patcher.patch(entry.getName(), InstrumentUtil.readAllBytes(is)); out.add(ResourcePoolEntry.create("/java.base/" + entry.getName(), content)); @@ -104,4 +99,36 @@ private void packJar(ResourcePoolBuilder out, File element, Set packages } } } + + private byte[] transformBaseModuleInfo(InputStream in, Set packages) { + try { + ClassNode classNode = new ClassNode(); + ClassReader cr = new ClassReader(in); + Attribute[] attributes = new Attribute[] { + new ModuleTargetAttribute(), new ModuleResolutionAttribute(), new ModuleHashesAttribute() + }; + cr.accept(classNode, attributes, 0); + // Add exports + for (String packageName : packages) { + classNode.module.exports.add(new ModuleExportNode(packageName, 0, null)); + } + // Add packages + classNode.module.packages.addAll(packages); + ClassWriter cw = new ClassWriter(0); + classNode.accept(cw); + return cw.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] findEntry(ResourcePool pool, String path) { + ResourcePoolEntry entry = pool.findEntry(path) + .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for: " + path)); + try (InputStream in = entry.content()) { + return InstrumentUtil.readAllBytes(in); + } catch (IOException e) { + throw new RuntimeException("Unable to read entry for: " + path, e); + } + } } diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java new file mode 100644 index 000000000..406049b16 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java @@ -0,0 +1,7 @@ +package edu.gmu.swe.phosphor.instrument; + +import java.io.IOException; + +public interface Patcher { + byte[] patch(String classFileName, byte[] classFileBuffer) throws IOException; +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java new file mode 100644 index 000000000..e7b0de35b --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java @@ -0,0 +1,91 @@ +package edu.gmu.swe.phosphor.instrument; + +import edu.columbia.cs.psl.phosphor.*; +import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.function.Function; + +/** + * Instances of this class are created via reflection. + */ +@SuppressWarnings("unused") +public class PhosphorInstrumentation implements Instrumentation { + private PreMain.PCLoggingTransformer transformer; + private Set classPathElements; + + @Override + public void configure(File source, Properties options) throws IOException { + options.put("java8", String.valueOf(!InstrumentUtil.isModularJvm(source))); + String[] arguments = OptionsUtil.createPhosphorMainArguments(options); + setUpClassLoader(source); + PhosphorOption.configure(false, arguments); + Configuration.init(); + TaintTrackingClassVisitor.IS_RUNTIME_INST = false; + transformer = new PreMain.PCLoggingTransformer(); + classPathElements = new HashSet<>(); + classPathElements.add(InstrumentUtil.getClassPathElement(PreMain.class)); + OptionsUtil.getConfigurationClasses(options).stream() + .map(InstrumentUtil::getClassPathElement) + .forEach(classPathElements::add); + } + + @Override + public java.util.Set getClassPathElements() { + return classPathElements; + } + + @Override + public byte[] apply(byte[] classFileBuffer) { + return transformer.transform(null, null, null, null, classFileBuffer, false); + } + + @Override + public Patcher createPatcher(Function entryLocator) { + String path = "/java.base/jdk/internal/misc/Unsafe.class"; + PhosphorPatcher patcher = new PhosphorPatcher(entryLocator.apply(path)); + return patcher::patch; + } + + @Override + public boolean shouldPack(String classFileName) { + return !classFileName.startsWith("edu/columbia/cs/psl/jigsaw/phosphor/instrumenter") + && !classFileName.endsWith("module-info.class") + && !classFileName.startsWith("org/") + && !classFileName.startsWith("edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported") + && classFileName.endsWith(".class"); + } + + @Override + public Set getElementsToPack() { + return classPathElements; + } + + private static void setUpClassLoader(File input) throws IOException { + List urls = new ArrayList<>(); + if (input.isDirectory()) { + Files.walkFileTree(input.toPath(), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().endsWith(".jar")) { + urls.add(file.toUri().toURL()); + } + return FileVisitResult.CONTINUE; + } + }); + } else if (input.getName().endsWith(".jar")) { + urls.add(input.toURI().toURL()); + } + urls.add(input.toURI().toURL()); + PreMain.bigLoader = new URLClassLoader(urls.toArray(new URL[0]), Instrumenter.class.getClassLoader()); + } +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java new file mode 100644 index 000000000..1f4715bc7 --- /dev/null +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java @@ -0,0 +1,21 @@ +package edu.gmu.swe.phosphor.instrument; + +import java.io.File; +import java.io.IOException; + +public final class PhosphorInstrumenter { + private PhosphorInstrumenter() { + throw new AssertionError("Tried to instantiate static utility class: " + getClass()); + } + + public static void main(String[] args) throws IOException { + // TODO + File source = new File(args[0]); + File target = new File(args[1]); + boolean verbose = Boolean.parseBoolean(args[2]); + String modules = args[3]; + System.out.printf("Instrumenting %s to %s%n", source, target); + long elapsedTime = Instrumenter.instrument(source, target, null, null, verbose, modules); + System.out.printf("Finished generation after %dms%n", elapsedTime); + } +} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java deleted file mode 100644 index 296c3da35..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentDriver.java +++ /dev/null @@ -1,54 +0,0 @@ -package edu.gmu.swe.phosphor.jlink; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public final class InstrumentDriver { - private InstrumentDriver() { - throw new AssertionError("Tried to instantiate static driver class: " + getClass()); - } - - /** - * Usage: java -cp <CLASS_PATH> edu.gmu.swe.phosphor.jlink.InstrumentDriver - * [OPTIONS] - * <INPUT_DIRECTORY> - * <OUTPUT_DIRECTORY> - * java -cp [CLASS-PATH] -jar phosphor.jar [OPTIONS] [input] [output] - * output_directory transformer_class core_jar [java_home] - *

- * output_directory: directory to which the instrumented JVM should be written. - */ - public static void main(String[] args) throws IOException, ClassNotFoundException { - File inputDirectory = new File(args[0]); - File outputDirectory = new File(args[1]); - Class transformerClass = Class.forName(args[1]); - File coreJar = new File(args[2]); - File javaHome = args.length == 3 ? new File(System.getProperty("java.home")) : new File(args[3]); - System.out.printf("Instrumenting %s to %s%n", javaHome, outputDirectory); - long elapsedTime = instrument(inputDirectory, outputDirectory, null); - System.out.printf("Finished generation after %dms%n", elapsedTime); - } - - public static long instrument(File inputDirectory, File outputDirectory, Instrumentation instrumentation) - throws IOException { - if (outputDirectory.exists()) { - throw new IllegalArgumentException("Output directory exists."); - } - if (!inputDirectory.isDirectory()) { - throw new IllegalArgumentException("Invalid input directory: " + inputDirectory); - } - long startTime = System.currentTimeMillis(); - try { - if (InstrumentUtil.isModularJvm(inputDirectory)) { - // TODO - // Arrays.asList("--add-modules", "ALL-MODULE-PATH") - } else { - new Instrumenter(instrumentation).process(inputDirectory, outputDirectory); - } - return System.currentTimeMillis() - startTime; - } catch (IOException | InterruptedException | ExecutionException e) { - throw new IOException("Failed to generate instrumented JVM", e); - } - } -} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java deleted file mode 100644 index b7bfdf935..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/InstrumentJLinkPlugin.java +++ /dev/null @@ -1,128 +0,0 @@ -package edu.gmu.swe.phosphor.jlink; - -import jdk.tools.jlink.plugin.Plugin; -import jdk.tools.jlink.plugin.ResourcePool; -import jdk.tools.jlink.plugin.ResourcePoolBuilder; -import jdk.tools.jlink.plugin.ResourcePoolEntry; -import org.objectweb.asm.Attribute; -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.commons.ModuleHashesAttribute; -import org.objectweb.asm.commons.ModuleResolutionAttribute; -import org.objectweb.asm.commons.ModuleTargetAttribute; -import org.objectweb.asm.tree.ClassNode; -import org.objectweb.asm.tree.ModuleExportNode; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.lang.instrument.ClassFileTransformer; -import java.lang.instrument.IllegalClassFormatException; -import java.util.Enumeration; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -public class InstrumentJLinkPlugin implements Plugin { - private ClassFileTransformer transformer; - private File coreJar; - - @Override - public String getName() { - return "phosphor-instrument"; - } - - @Override - public Category getType() { - return Category.FILTER; - } - - @Override - public String getDescription() { - return "Applies instrumentation to the runtime image and packs core classes into the java.base module"; - } - - @Override - public boolean hasArguments() { - return true; - } - - @Override - public void configure(Map config) { - coreJar = new File(config.get("core")); - try { - transformer = (ClassFileTransformer) Class.forName(config.get("transformer")) - .getDeclaredConstructor() - .newInstance(); - } catch (ReflectiveOperationException e) { - e.printStackTrace(); - throw new IllegalArgumentException("Failed to create transformer of type: " + config.get("transformer"), e); - } - } - - @Override - public ResourcePool transform(ResourcePool in, ResourcePoolBuilder out) { - in.transformAndCopy(e -> transform(e, out), out); - return out.build(); - } - - private ResourcePoolEntry transform(ResourcePoolEntry entry, ResourcePoolBuilder out) { - if (entry.type().equals(ResourcePoolEntry.Type.CLASS_OR_RESOURCE)) { - if (entry.path().endsWith("module-info.class") && entry.path().startsWith("/java.base")) { - // Transform java.base's module-info.class file and pack core classes into java.base - return entry.copyWithContent(transformBaseModuleInfo(entry.content(), out)); - } else if (entry.path().endsWith(".class")) { - try { - byte[] instrumented = transformer.transform(null, null, null, - null, entry.contentBytes()); - return instrumented == null ? entry : entry.copyWithContent(instrumented); - } catch (IllegalClassFormatException e) { - return entry; - } - } - } - return entry; - } - - private Set packCore(ResourcePoolBuilder out) throws IOException { - Set packages = new HashSet<>(); - try (ZipFile zip = new ZipFile(coreJar)) { - Enumeration entries = zip.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - if (entry.getName().endsWith(".class")) { - try (InputStream is = zip.getInputStream(entry)) { - out.add(ResourcePoolEntry.create("/java.base/" + entry.getName(), is.readAllBytes())); - } - packages.add(entry.getName().substring(0, entry.getName().lastIndexOf('/'))); - } - } - } - return packages; - } - - private byte[] transformBaseModuleInfo(InputStream in, ResourcePoolBuilder out) { - try { - Set packages = packCore(out); - ClassNode classNode = new ClassNode(); - ClassReader cr = new ClassReader(in); - Attribute[] attributes = new Attribute[]{new ModuleTargetAttribute(), - new ModuleResolutionAttribute(), - new ModuleHashesAttribute()}; - cr.accept(classNode, attributes, 0); - // Add exports - for (String packageName : packages) { - classNode.module.exports.add(new ModuleExportNode(packageName, 0, null)); - } - // Add packages - classNode.module.packages.addAll(packages); - ClassWriter cw = new ClassWriter(0); - classNode.accept(cw); - return cw.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java deleted file mode 100644 index 6da2e80d1..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/Instrumentation.java +++ /dev/null @@ -1,30 +0,0 @@ -package edu.gmu.swe.phosphor.jlink; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; - -/** - * Implementors are expected to have a zero-argument, public constructor. - */ -public interface Instrumentation { - /** - * Configures this instance using the specified options. - * @param inputDirectory the directory to be instrumented - * @param outputDirectory the directory to which the instrumented output should be written - * @param options the key-value pairs that should be used to configure this instance - * @throws IOException if an I/O error occurs - */ - void configure(File inputDirectory, File outputDirectory, Properties options) - throws IOException, ReflectiveOperationException; - - /** - * Returns an array of class path elements needed to use this class. - * The returned array should be non-null. - * All elements of the returned array should be non-null. - * @return class path elements needed to use this class - */ - File[] getClassPathElements(); - - byte[] apply(byte[] classFileBuffer); -} diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java deleted file mode 100644 index e3c87ef2e..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/JLinkInvoker.java +++ /dev/null @@ -1,68 +0,0 @@ -package edu.gmu.swe.phosphor.jlink; - -import edu.columbia.cs.psl.phosphor.OptionsUtil; - -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -public final class JLinkInvoker { - private JLinkInvoker() { - throw new AssertionError(); - } - - public static void invoke( - File javaHome, File outputDirectory, List jlinkOptions, Class transformerClass, File coreJar) - throws InterruptedException, IOException { - Properties properties = new Properties(); - String jlinkAgentJar = - InstrumentUtil.getClassPathElement(JLinkRegistrationAgent.class).getAbsolutePath(); - List command = new ArrayList<>(); - command.add(InstrumentUtil.javaHomeToJLinkExec(javaHome).getAbsolutePath()); - command.add("-J-javaagent:" + jlinkAgentJar); - command.add("-J--class-path=" + buildClassPath(transformerClass, properties)); - command.add("-J--add-reads=" + JLinkRegistrationAgent.MODULE_NAME + "=ALL-UNNAMED"); - command.add("-J--module-path=" + jlinkAgentJar); - command.add("-J--add-modules=" + JLinkRegistrationAgent.MODULE_NAME); - command.add(getPluginOption(transformerClass, coreJar)); - command.add("--output=" + outputDirectory.getAbsolutePath()); - command.addAll(processJLinkOptions(jlinkOptions)); - ProcessBuilder builder = new ProcessBuilder(command); - System.out.println(String.join(" ", builder.command())); - Process process = builder.inheritIO().start(); - if (process.waitFor() != 0) { - throw new RuntimeException("Failed to create instrumented runtime image"); - } - } - - private static String buildClassPath(Class transformerClass, Properties properties) { - Set> classes = OptionsUtil.getConfigurationClasses(properties); - classes.add(transformerClass); - return classes.stream() - .map(edu.columbia.cs.psl.jigsaw.phosphor.instrumenter.JLinkInvoker::getClassPathElement) - .map(File::getAbsolutePath) - .collect(Collectors.joining(File.pathSeparator)); - } - - private static List processJLinkOptions(List jlinkOptions) { - // Ensure that all required modules are included - jlinkOptions = new ArrayList<>(jlinkOptions); - Set modules = - new LinkedHashSet<>(Arrays.asList("java.base", "jdk.jdwp.agent", "java.instrument", "jdk.unsupported")); - int i = jlinkOptions.indexOf("--add-modules"); - if (i != -1) { - modules.addAll(Arrays.asList(jlinkOptions.get(i + 1).split(","))); - jlinkOptions.set(i + 1, String.join(",", modules)); - } else { - jlinkOptions.add("--add-modules"); - jlinkOptions.add(String.join(",", modules)); - } - return jlinkOptions; - } - - private static String getPluginOption(Class transformerClass, File coreJar) { - return String.format( - "--phosphor-instrument=x:transformer=%s:core=%s", transformerClass.getName(), coreJar.getAbsolutePath()); - } -} \ No newline at end of file diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java deleted file mode 100644 index 4f7d211b0..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/jlink/PhosphorInstrumentation.java +++ /dev/null @@ -1,35 +0,0 @@ -package edu.gmu.swe.phosphor.jlink; - -import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.OptionsUtil; -import edu.columbia.cs.psl.phosphor.PhosphorOption; -import edu.columbia.cs.psl.phosphor.PreMain; -import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; - -import java.io.File; -import java.util.Properties; - -public class PhosphorInstrumentation implements Instrumentation { - private PreMain.PCLoggingTransformer transformer; - - @Override - public void configure(File inputDirectory, File outputDirectory, Properties options) { - String[] arguments = OptionsUtil.createPhosphorMainArguments(inputDirectory, outputDirectory, options); - PhosphorOption.configure(false, arguments); - Configuration.init(); - TaintTrackingClassVisitor.IS_RUNTIME_INST = false; - transformer = new PreMain.PCLoggingTransformer(); - } - - @Override - public File[] getClassPathElements() { - return new File[] {InstrumentUtil.getClassPathElement(PreMain.class)}; - } - - @Override - public byte[] apply(byte[] classFileBuffer) { - transformer.transform(null, null, null, null, - classFileBuffer, false); - return null; - } -} diff --git a/phosphor-instrument-jigsaw/src/main/java/module-info.java b/phosphor-instrument-jigsaw/src/main/java/module-info.java index 5d368d70a..e154ca28a 100644 --- a/phosphor-instrument-jigsaw/src/main/java/module-info.java +++ b/phosphor-instrument-jigsaw/src/main/java/module-info.java @@ -1,5 +1,5 @@ -module edu.gmu.swe.phosphor.jlink { - exports edu.gmu.swe.phosphor.jlink; +module edu.gmu.swe.phosphor.instrument { + exports edu.gmu.swe.phosphor.instrument; requires jdk.jlink; requires java.instrument; } diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java similarity index 53% rename from phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java rename to phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java index c2f594c40..77d7fcd1c 100644 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentingMojo.java +++ b/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java @@ -1,9 +1,5 @@ package edu.gmu.swe.phosphor.instrument; -import edu.gmu.swe.phosphor.jlink.DeletingFileVisitor; -import edu.gmu.swe.phosphor.jlink.InstrumentDriver; -import edu.gmu.swe.phosphor.jlink.InstrumentUtil; -import edu.gmu.swe.phosphor.jlink.Instrumentation; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -18,32 +14,32 @@ import java.util.Properties; /** - * Creates an instrumented Java installation. + * Creates an instrumented Java installation (i.e., Java Development Kit or Java Runtime Environment). *

- * If {@link InstrumentingMojo#outputDirectory} does not exists, a new instrumented Java installation is created. - * If {@link InstrumentingMojo#outputDirectory} exists and is not a Java installation previously created by + * If {@link InstrumentMojo#outputDirectory} does not exists, a new instrumented Java installation is created. + * If {@link InstrumentMojo#outputDirectory} exists and is not a Java installation previously created by * this plugin, this plugin will throw a {@link MojoExecutionException}. - * If {@link InstrumentingMojo#outputDirectory} exists and is a Java installation previously created by this plugin, + * If {@link InstrumentMojo#outputDirectory} exists and is a Java installation previously created by this plugin, * this plugin checks whether the Java installation needs to be recreated. - * If the {@link InstrumentingMojo#forceCreation} is {@code false} and the existing Java installation was created from - * the same {@link InstrumentingMojo#javaHome uninstrumented Java installation} using the same - * {@link InstrumentingMojo#instrumentationType} and {@link InstrumentingMojo#options}, then the Java installation does + * If the {@link InstrumentMojo#forceCreation} is {@code false} and the existing Java installation was created from + * the same {@link InstrumentMojo#javaHome uninstrumented Java installation} using the same + * {@link InstrumentMojo#instrumentationType} and {@link InstrumentMojo#options}, then the Java installation does * not need to be recreated and the plugin terminates. * Otherwise, this plugin will delete the existing Java installation and any - * {@link InstrumentingMojo#linkedCaches linked files or directories}. + * {@link InstrumentMojo#linkedCaches linked files or directories}. * Then, this plugin will create a new instrumented Java installation. *

* The instrumented Java installation is created by instrumenting the Java installation located in the directory - * {@link InstrumentingMojo#javaHome} or the Java installation used to run the Maven process if - * {@link InstrumentingMojo#javaHome} was not specified. - * An instance of the specified {@link InstrumentingMojo#instrumentationType instrumentation class} is created and - * configured using the specified {@link InstrumentingMojo#options}. + * {@link InstrumentMojo#javaHome} or the Java installation used to run the Maven process if + * {@link InstrumentMojo#javaHome} was not specified. + * An instance of the specified {@link InstrumentMojo#instrumentationType instrumentation class} is created and + * configured using the specified {@link InstrumentMojo#options}. * This instance determines the type of instrumentation applied. * - * @see edu.gmu.swe.phosphor.jlink.Instrumentation + * @see edu.gmu.swe.phosphor.instrument.Instrumentation */ @Mojo(name = "instrument", defaultPhase = LifecyclePhase.PROCESS_TEST_RESOURCES) -public class InstrumentingMojo extends AbstractMojo { +public class InstrumentMojo extends AbstractMojo { /** * Directory where the Java installation to be instrumented is located. * If not specified, then the Java installation used to run the Maven process will be used. @@ -80,7 +76,7 @@ public class InstrumentingMojo extends AbstractMojo { */ @Parameter( property = "phosphor.instrumentationType", - defaultValue = "edu.gmu.swe.phosphor.jlink.PhosphorInstrumentation") + defaultValue = "edu.gmu.swe.phosphor.instrument.PhosphorInstrumentation") private String instrumentationType; /** * Options passed to {@link Instrumentation#configure}. @@ -88,19 +84,16 @@ public class InstrumentingMojo extends AbstractMojo { @Parameter(property = "phosphor.options") private Properties options = new Properties(); /** - * File used to store the options used by this plugin to create the instrumented Java installation + * A comma-separated list of Java modules to include in the instrumented Java installation. + * Used only for instrumenting Java 9+ installations. */ - private File optionsFile; + @Parameter(property = "phosphor.modules", defaultValue = "ALL-MODULE-PATH") + private String modules; /** - * File used to store the checksum for the instrumentation used by this plugin to create the instrumented Java - * installation + * True is information about instrumentation progress should be logged. */ - private File checksumFile; - /** - * File used to store the path of the uninstrumented Java installation and fully qualified name of the - * implementation of {@link Instrumentation} used by this plugin to create the instrumented Java installation - */ - private File infoFile; + @Parameter(property = "phosphor.verbose", defaultValue = "false") + private boolean verbose; /** * Creates an instrumented Java installation. @@ -112,17 +105,12 @@ public void execute() throws MojoExecutionException { if (!InstrumentUtil.isJavaHome(javaHome)) { throw new MojoExecutionException("Expected Java installation at: " + javaHome); } - optionsFile = new File(outputDirectory, "phosphor-instrument" + File.separator + "option.properties"); - checksumFile = new File(optionsFile.getParent(), "checksum.md5"); - infoFile = new File(optionsFile.getParent(), "info.txt"); Instrumentation instance = createInstrumentation(); byte[] checksum = computeChecksum(instance); String info = String.format("%s%n%s", javaHome.getAbsolutePath(), instrumentationType); - if (InstrumentUtil.isJavaHome(outputDirectory) - && checksumFile.isFile() - && optionsFile.isFile() - && infoFile.isFile()) { - if (!forceCreation && checkMatchFiles(checksum, info)) { + MatchInfo match = new MatchInfo(outputDirectory); + if (InstrumentUtil.isJavaHome(outputDirectory) && match.exists()) { + if (!forceCreation && match.check(checksum, info, options)) { getLog().info("Existing instrumented Java installation with correct settings found: " + outputDirectory); getLog().info("Skipping creation."); @@ -130,47 +118,31 @@ public void execute() throws MojoExecutionException { if (!forceCreation) { getLog().info("Existing Java installation did not have correct settings."); } - getLog().info("Recreating Java installation : " + outputDirectory); - deleteExistingJdkAndLinkedCaches(); - createInstrumentedJdk(instance, checksum, info); + deleteExisting(); + instrument(instance, checksum, info, match); } } else if (outputDirectory.exists()) { String message = "Failed to create instrumented Java installation." + " %s already exists and is not an instrumented Java installation."; throw new MojoExecutionException(String.format(message, outputDirectory)); } else { - getLog().info("Creating Java installation : " + outputDirectory); - createInstrumentedJdk(instance, checksum, info); + instrument(instance, checksum, info, match); } } private Instrumentation createInstrumentation() throws MojoExecutionException { try { - Class clazz = Class.forName(instrumentationType, true, getClass().getClassLoader()); - Instrumentation instance = (Instrumentation) clazz.getConstructor().newInstance(); - instance.configure(javaHome, outputDirectory, options); - return instance; + return Instrumentation.create(instrumentationType, javaHome, options); } catch (ClassCastException | IOException | ReflectiveOperationException e) { throw new MojoExecutionException( "Error occurred while creating instrumentation instance: " + instrumentationType, e); } } - private boolean checkMatchFiles(byte[] checksum, String info) throws MojoExecutionException { - try (FileReader reader = new FileReader(optionsFile)) { - Properties foundOptions = new Properties(); - foundOptions.load(reader); - return foundOptions.equals(options) - && Arrays.equals(checksum, InstrumentUtil.readAllBytes(checksumFile)) - && new String(InstrumentUtil.readAllBytes(infoFile)).equals(info); - } catch (IOException e) { - throw new MojoExecutionException("Failed to read match info", e); - } - } - - private void deleteExistingJdkAndLinkedCaches() throws MojoExecutionException { + private void deleteExisting() throws MojoExecutionException { try { Files.walkFileTree(outputDirectory.toPath(), new DeletingFileVisitor()); + getLog().info("Deleted existing Java installation: " + outputDirectory); for (File file : linkedCaches) { if (file.exists()) { Files.walkFileTree(file.toPath(), new DeletingFileVisitor()); @@ -182,27 +154,17 @@ private void deleteExistingJdkAndLinkedCaches() throws MojoExecutionException { } } - private void createInstrumentedJdk(Instrumentation instance, byte[] checksum, String info) + private void instrument(Instrumentation instance, byte[] checksum, String info, MatchInfo match) throws MojoExecutionException { + getLog().info("Creating Java installation: " + outputDirectory); try { - InstrumentDriver.instrument(javaHome, outputDirectory, instance); + long elapsedTime = + Instrumenter.instrument(javaHome, outputDirectory, options, instance, verbose, modules); + getLog().info(String.format("Finished creating instrumented Java installation after %d ms", elapsedTime)); } catch (IOException e) { - throw new MojoExecutionException("Failed to instrument Java instrumentation.", e); - } - writeMatchFiles(checksum, info); - } - - private void writeMatchFiles(byte[] checksum, String info) throws MojoExecutionException { - try { - InstrumentUtil.ensureDirectory(optionsFile.getParentFile()); - Files.write(checksumFile.toPath(), checksum); - Files.write(infoFile.toPath(), info.getBytes()); - try (FileWriter writer = new FileWriter(optionsFile)) { - options.store(writer, null); - } - } catch (IOException e) { - throw new MojoExecutionException("Failed to write match files", e); + throw new MojoExecutionException("Failed to create instrumented Java instrumentation.", e); } + match.write(checksum, info, options); } private static byte[] computeChecksum(Instrumentation instrumentation) throws MojoExecutionException { @@ -218,4 +180,61 @@ private static byte[] computeChecksum(Instrumentation instrumentation) throws Mo throw new MojoExecutionException("Failed to compute instrumentation checksum", e); } } + + private static final class MatchInfo { + /** + * File used to store the options used to instrument a location. + *

+ * Non-null. + */ + private final File optionsFile; + /** + * File used to store the checksum for the class path of the instrumentation used to instrument a + * location. + *

+ * Non-null. + */ + private final File checksumFile; + /** + * File used to store the path of the source location instrumented and the fully qualified name of the + * implementation of {@link Instrumentation} used to instrument a location. + */ + private final File infoFile; + + public MatchInfo(File directory) { + File parent = new File(directory, "phosphor-instrument-match"); + this.optionsFile = new File(parent, "options.properties"); + this.checksumFile = new File(parent, "class-path.md5"); + this.infoFile = new File(parent, "info.txt"); + } + + public boolean exists() { + return optionsFile.isFile() && checksumFile.isFile() && infoFile.isFile(); + } + + private boolean check(byte[] checksum, String info, Properties options) throws MojoExecutionException { + try (FileReader reader = new FileReader(optionsFile)) { + Properties foundOptions = new Properties(); + foundOptions.load(reader); + return foundOptions.equals(options) + && Arrays.equals(checksum, InstrumentUtil.readAllBytes(checksumFile)) + && new String(InstrumentUtil.readAllBytes(infoFile)).equals(info); + } catch (IOException e) { + throw new MojoExecutionException("Failed to read match info", e); + } + } + + public void write(byte[] checksum, String info, Properties options) throws MojoExecutionException { + try { + InstrumentUtil.ensureDirectory(optionsFile.getParentFile()); + Files.write(checksumFile.toPath(), checksum); + Files.write(infoFile.toPath(), info.getBytes()); + try (FileWriter writer = new FileWriter(optionsFile)) { + options.store(writer, null); + } + } catch (IOException e) { + throw new MojoExecutionException("Failed to write match info", e); + } + } + } } From 0535a2f5b82d8cc87a0a5a5facd9844da5a03255 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sat, 25 Nov 2023 16:17:28 -0500 Subject: [PATCH 03/12] * Fixed issue with GHA --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8dad8ce55..7fc503a22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Run tests - run: mvn install -ntp -Pdacapo + run: mvn install -ntp -Pdacapo -pl :integration-tests deploy: runs-on: self-hosted needs: build-and-test From 8730d6cff0f2fd625e8caec58787358bc924c337 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sat, 25 Nov 2023 16:31:13 -0500 Subject: [PATCH 04/12] * Removed accidental debugger java option from JLinkInvoker --- .../main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java index c67a485c4..7f1017ba6 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java @@ -21,7 +21,6 @@ public static void invoke( String classPath = buildClassPath(instrumentation); command.add(InstrumentUtil.javaHomeToJLinkExec(javaHome).getAbsolutePath()); command.add("-J-javaagent:" + jlinkAgentJar); - command.add("-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005"); command.add("-J--class-path=" + classPath); command.add("-J--add-reads=" + JLinkRegistrationAgent.MODULE_NAME + "=ALL-UNNAMED"); command.add("-J--module-path=" + jlinkAgentJar); From 82c3551f1924d96ee0d3589ec0153a1a95f5499e Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sat, 25 Nov 2023 17:25:55 -0500 Subject: [PATCH 05/12] * Started updating README --- README.md | 244 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 193 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index e38893c6e..7c95d25f0 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,110 @@ -Phosphor: Dynamic Taint Tracking for the JVM -======== - - -Phosphor is a system for performing dynamic taint analysis in the JVM, on commodity JVMs (e.g. Oracle's HotSpot or OpenJDK's IcedTea). This repository contains the source for Phosphor. For more information about how Phosphor works and what it could be useful for, please refer to our [OOPSLA 2014 paper](http://jonbell.net/publications/phosphor), [ISSTA 2015 Tool Demo ](http://mice.cs.columbia.edu/getTechreport.php?techreportID=1601) or email [Jonathan Bell](mailto:jbell@cs.columbia.edu). José Cambronero also maintains a [series of examples on using Phosphor](https://github.com/josepablocam/phosphor-examples/). - -Phosphor has been extensively developed since its original publication, and now includes many features and options not described in the OOPSLA 2014 paper. If you are looking to replicate our OOPSLA 2014 experiments, the easiest way to get the same version with certainty is to use this [VM Image with all relevant files here](http://academiccommons.columbia.edu/catalog/ac%3A182689), and to follow the [README with instructions for doing so here](https://www.dropbox.com/s/dmebj6k8izams6p/artifact-63-phosphor.pdf?dl=0). - -### Refactoring Status and Roadmap -This branch contains what is nearly a complete rewrite of Phosphor, using a less fragile (but slower) approach to pass taint tags between methods. It also is the first version of Phosphor to support Java 9+. +# Phosphor: Dynamic Taint Tracking for the JVM + +Phosphor is a system for performing dynamic taint analysis in the JVM, on commodity JVMs (e.g. Oracle's HotSpot or +OpenJDK's IcedTea). +Phosphor associates labels, which we referred to as taint tags, with program data and propagates these +labels along information "flows" at runtime. +This repository contains the source code for Phosphor. +For more information about how Phosphor works and its uses, please refer to +our [OOPSLA 2014 paper](http://jonbell.net/publications/phosphor), +[ISSTA 2015 Tool Demo](http://mice.cs.columbia.edu/getTechreport.php?techreportID=1601) +or email [Jonathan Bell](mailto:jbell@cs.columbia.edu). +José Cambronero also maintains +a [series of examples on using Phosphor](https://github.com/josepablocam/phosphor-examples/). + +Phosphor has been extensively developed since its original publication, and now includes many features and options not +described in the OOPSLA 2014 paper. +If you are looking to replicate our OOPSLA 2014 experiments, the easiest way to get +the same version with certainty is to use +this [VM Image with all relevant files here](http://academiccommons.columbia.edu/catalog/ac%3A182689), and to follow the +[README with instructions for doing so here](https://www.dropbox.com/s/dmebj6k8izams6p/artifact-63-phosphor.pdf?dl=0). + +Phosphor currently requires Java 9+ to build, but it can also be used on Java 8. + +## Refactoring Status and Roadmap + +This branch contains what is nearly a complete rewrite of Phosphor, using a less fragile (but slower) approach to pass +taint tags between methods. +It also is the first version of Phosphor to support Java 9+. Remaining tasks for this branch before promotion: + * Implement control tracking semantics (currently entirely unimplemented) * Performance optimization * Improve documentation * Consider simplifying distribution of the now multiple jars, or alternatively remove the phosphor-distribution module * Consider removing the PHOSPHOR_TAG field from objects, and remove `MultiTainter.taintedObject` +## Building + +### Requirements + +* Java Development Kit (JDK) 9+ +* [Apache Maven](https://maven.apache.org/) 3.6.0+ + +### Steps + +1. Clone or download this repository. +2. Ensure that some version of the JDK 9+ is installed. + A JDK can be downloads from [Oracle](https://www.oracle.com/java/technologies/downloads/) or + the [Adoptium Working Group](https://adoptium.net/temurin/releases/). +3. Set the JAVA_HOME environmental variable to the path of this JDK installation. + On Linux and Mac this can be done by running `export JAVA_HOME=`, where <PATH-TO-JDK> is the path + of the JDK installation. +4. Ensure that you have installed Apache Maven 3.6.0+. + Directions for [downloading](https://maven.apache.org/download.cgi) + and [installing](https://maven.apache.org/install.html) Maven are available on the project page for Maven. +5. In the root directory of this project (the one where this README file is located), run `mvn -DskipTests install`. + +## Running Phosphor's Tests + +Once you have built Phosphor according to the directions described above in the +Section ["Building Phosphor"](#Building-Phosphor), you can run Phosphor's tests and examples. +Although Phosphor currently requires Java 9+ to build, it can also be used on Java 8. +If you would like to run Phosphor's tests on Java 8, build Phosphor using Java 9+, then change the +JAVA_HOME environmental variable to the path of a JDK 8 installation before running the tests. + +TODO what command should you run and what will this command do. +TODO Describe some of the tests + +## Running the DaCapo Benchmarks + +## Creating an Instrumented Java Installation + +In order to associate labels with data and to propagate these labels, Phosphor uses Java bytecode instrumentation to +modify Java classes. +In order to track the flow of information through classes in the Java Class Library (JCL), such as `java.lang.String` +and `java.util.List`, Phosphor must also instrument the bytecode of JCL classes. +Therefore, the first step when using Phosphor is to create an instrumented Java installation +(i.e., Java Development Kit or Java Runtime Environment). +A Java installation can be downloaded from [Oracle](https://www.oracle.com/java/technologies/downloads/) or +the [Adoptium Working Group](https://adoptium.net/temurin/releases/). +Once you have obtained a Java installation, it can be instrumented either using Phosphor's [driver JAR](#Driver-Jar) or +[Maven plugin](#Maven-Plugin). +We discuss both options and how to configure Phosphor below. -Running -------- -Phosphor works by modifying your application's bytecode to perform data flow tracking. To be complete, Phosphor also modifies the bytecode of JRE-provided classes, too. The first step to using Phosphor is generating an instrumented version of your runtime environment. We have tested Phosphor with Oracle and OpenJDK Java 8 runtimes. +### Driver JAR -The instrumenter takes two primary arguments: first a path containing the classes to instrument, and then a destination for the instrumented classes. You can also specify to track taint tags through control flow, to use objects as tags (instead of integers), or to automatically perform taint marking in particular methods using the various options as shown by invoking Phosphor with the "-help" option. +### Maven Plugin +### Configuring Phosphor + +## Running an Application with Phosphor + +## Extending Phosphor + +## Running + +Phosphor works by modifying your application's bytecode to perform data flow tracking. +To be complete, Phosphor also modifies the bytecode of JRE-provided classes, too. +The first step to using Phosphor is generating an instrumented version of your runtime environment. +We have tested Phosphor with Oracle and OpenJDK Java 8 runtimes. + +The instrumenter takes two primary arguments: first a path containing the classes to instrument, and then a destination +for the instrumented classes. +You can also specify to track taint tags through control flow, to use objects as tags ( +instead of integers), or to automatically perform taint marking in particular methods using the various options as shown +by invoking Phosphor with the "-help" option. ``` usage: java -jar phosphor.jar [OPTIONS] [input] [output] @@ -32,85 +114,145 @@ usage: java -jar phosphor.jar [OPTIONS] [input] [output] (on by default) ``` -Phosphor now should be configured to correctly run JUnit tests (with taint tracking) in most environments (Mac + Linux + Windows). Running `mvn verify` should cause Phosphor to generate several different instrumented JRE's (for multitaint use, int-tag taint use, and control track use) into the project's `target` directory, then run unit tests in that JRE that are automatically tracked. You take a look at the test cases to see some example usage. Tests that end in `ObjTagITCase` are executed with Phosphor configured for object tags (multi tainting), and `ImplicitITCase` tests run in the control tracking mode. +Phosphor now should be configured to correctly run JUnit tests (with taint tracking) in most environments (Mac + Linux + +Windows). +Running `mvn verify` should cause Phosphor to generate several different instrumented JRE's (for multitaint +use, int-tag taint use, and control track use) into the project's `target` directory, then run unit tests in that JRE +that are automatically tracked. +You take a look at the test cases to see some example usage. +Tests that end +in `ObjTagITCase` are executed with Phosphor configured for object tags (multi tainting), and `ImplicitITCase` tests run +in the control tracking mode. ### Instrumenting a Java 9+ VM -We'll assume that in all of the code examples below, we're in the same directory as this README.md, that you have already run `mvn package`, and that the JVM is located here: `/Library/Java/JavaVirtualMachines/adoptopenjdk-16.jdk/Contents/Home` (modify this path in the commands below to match your environment). -For Java >= 9, we generate an instrumented JVM by using the `jlink` tool. We have created a jar (in `phosphor-instrument-jigsaw`) that wraps this entire process, invoking these tools automatically. To instrument the JVM, run this command: +We'll assume that in all of the code examples below, we're in the same directory as this README.md, that you have +already run `mvn package`, and that the JVM is located +here: `/Library/Java/JavaVirtualMachines/adoptopenjdk-16.jdk/Contents/Home` (modify this path in the commands below to +match your environment). + +For Java >= 9, we generate an instrumented JVM by using the `jlink` tool. We have created a jar ( +in `phosphor-instrument-jigsaw`) that wraps this entire process, invoking these tools automatically. To instrument the +JVM, run this command: `java -jar phosphor-instrument-jigsaw/target/phosphor-instrument-jigsaw-0.1.0-SNAPSHOT.jar /Library/Java/JavaVirtualMachines/adoptopenjdk-16.jdk/Contents/Home jre-inst` -The next step is to instrument the code which you would like to track. This time, we will use Phosphor's normal instrumenter (not the jigsaw JVM instrumenter), and will pass your entire (compiled) code base to Phosphor for instrumentation, and specify an output folder for that. Example: `java -jar Phosphor/target/Phosphor-0.1.0-SNAPSHOT.jar path-to-application output-path-for-instrumented-code` +The next step is to instrument the code which you would like to track. This time, we will use Phosphor's normal +instrumenter (not the jigsaw JVM instrumenter), and will pass your entire (compiled) code base to Phosphor for +instrumentation, and specify an output folder for that. +Example: `java -jar Phosphor/target/Phosphor-0.1.0-SNAPSHOT.jar path-to-application output-path-for-instrumented-code` We can now run the instrumented code using our instrumented JRE, as such: + ``` jre-inst/bin/java -javaagent:phosphor-jigsaw-javaagent/target/phosphor-jigsaw-javaagent-0.1.0-SNAPSHOT.jar -cp path-to-instrumented-code your.main.class ```` - ### Instrumenting a Java 8 VM -We'll assume that in all of the code examples below, we're in the same directory (which has a copy of Phosphor-0.0.5-SNAPSHOT.jar, which you generated by downloading Phosphor, and running `mvn package`), and that the JRE is located here: `/Library/Java/JavaVirtualMachines/jdk1.8.0_222.jdk/Contents/Home` (modify this path in the commands below to match your environment). -*Important note on OpenJDK vs Oracle's JDK:* Oracle's JVM requires that the jar that contains all of the cryptography routines (`jce.jar`) be signed by Oracle, for export control purposes. OpenJDK does not. When Phosphor instruments the JVM, it will break these signatures. Hence, it is not possible to use Phosphor with Oracle's JDK *and* use the cryptography functionality of the JVM without some extra work. The first option is (if you have been granted a signing key by Oracle), you can sign the resulting jar. The more attainable option is to instead use the OpenJDK `jce.jar` file with your Oracle JVM, which will not have the signature checks, and will work just fine. Alternatively, if this is too complicated (and you need cryptography support), you can just choose to use OpenJDK entirely (which does not have this check). +We'll assume that in all of the code examples below, we're in the same directory (which has a copy of +Phosphor-0.0.5-SNAPSHOT.jar, which you generated by downloading Phosphor, and running `mvn package`), and that the JRE +is located here: `/Library/Java/JavaVirtualMachines/jdk1.8.0_222.jdk/Contents/Home` (modify this path in the commands +below to match your environment). + +*Important note on OpenJDK vs Oracle's JDK:* Oracle's JVM requires that the jar that contains all of the cryptography +routines (`jce.jar`) be signed by Oracle, for export control purposes. OpenJDK does not. When Phosphor instruments the +JVM, it will break these signatures. Hence, it is not possible to use Phosphor with Oracle's JDK *and* use the +cryptography functionality of the JVM without some extra work. The first option is (if you have been granted a signing +key by Oracle), you can sign the resulting jar. The more attainable option is to instead use the OpenJDK `jce.jar` file +with your Oracle JVM, which will not have the signature checks, and will work just fine. Alternatively, if this is too +complicated (and you need cryptography support), you can just choose to use OpenJDK entirely (which does not have this +check). Then, to instrument the JRE we'll run: `java -jar Phosphor-0.1.0-SNAPSHOT.jar /Library/Java/JavaVirtualMachines/jdk1.8.0_222.jdk/Contents/Home jre-inst` After you do this, make sure to chmod +x the binaries in the new folder, e.g. `chmod +x jre-inst/bin/*` -The next step is to instrument the code which you would like to track. This time when you run the instrumenter, pass your entire (compiled) code base to Phosphor for instrumentation, and specify an output folder for that. +The next step is to instrument the code which you would like to track. This time when you run the instrumenter, pass +your entire (compiled) code base to Phosphor for instrumentation, and specify an output folder for that. We can now run the instrumented code using our instrumented JRE, as such: + ``` export JAVA_HOME=jre-inst/ $JAVA_HOME/bin/java -Xbootclasspath/a:Phosphor-0.1.0-SNAPSHOT.jar -javaagent:Phosphor-0.1.0-SNAPSHOT.jar -cp path-to-instrumented-code your.main.class ```` -Note: It is not 100% necessary to instrument your application/library code in advance - the javaagent will detect any uninstrumented class files as they are being loaded into the JVM and instrument them as necessary. If you want to do this, then you may want to add the flag `-javaagent:Phosphor-0.1.0-SNAPSHOT.jar=cacheDir=someCacheFolder` and Phosphor will cache the generated files in `someCacheFolder` so they aren't regenerated every run. If you take a look at the execution of Phosphor's JUnit tests, you'll notice that this is how they are instrumented. It's always necessary to instrument the JRE in advance though for bootstrapping. +Note: It is not 100% necessary to instrument your application/library code in advance - the javaagent will detect any +uninstrumented class files as they are being loaded into the JVM and instrument them as necessary. If you want to do +this, then you may want to add the flag `-javaagent:Phosphor-0.1.0-SNAPSHOT.jar=cacheDir=someCacheFolder` and Phosphor +will cache the generated files in `someCacheFolder` so they aren't regenerated every run. If you take a look at the +execution of Phosphor's JUnit tests, you'll notice that this is how they are instrumented. It's always necessary to +instrument the JRE in advance though for bootstrapping. -New 2/27/19: You can no longer specify auto taint methods (what were sources/sinks/taint through methods) for the static instrumenter. Instead, ALL autotaint instrumentation happens via the java agent (this makes it possible to detect child-classes of auto taint classes). You can specify the files to the java agent using the syntax `-javaagent:Phosphor-0.0.4-SNAPSHOT.jar=taintSources={taintSourceFile},taintSinks={taintSinksFile},taintThrough={taintThroughFile}` +New 2/27/19: You can no longer specify auto taint methods (what were sources/sinks/taint through methods) for the static +instrumenter. Instead, ALL autotaint instrumentation happens via the java agent (this makes it possible to detect +child-classes of auto taint classes). You can specify the files to the java agent using the +syntax `-javaagent:Phosphor-0.0.4-SNAPSHOT.jar=taintSources={taintSourceFile},taintSinks={taintSinksFile},taintThrough={taintThroughFile}` +## Interacting with Phosphor -Interacting with Phosphor ------ -Phosphor exposes a simple API to allow to marking data with tags, and to retrieve those tags. Key functionality is implemented in ``edu.columbia.cs.psl.phosphor.runtime.MultiTainter``. To get or set the taint tag of a primitive type, developers call the taintedX or getTaint(X) method (replacing X with each of the primitive types, e.g. taintedByte, etc.). +Phosphor exposes a simple API to allow to marking data with tags, and to retrieve those tags. Key functionality is +implemented in ``edu.columbia.cs.psl.phosphor.runtime.MultiTainter``. To get or set the taint tag of a primitive type, +developers call the taintedX or getTaint(X) method (replacing X with each of the primitive types, e.g. taintedByte, +etc.). Ignore the methods ending with the suffix $$PHOSPHOR, they are used internally. -To get or set the taint tag of an object, first cast that object to the interface TaintedWithObjTag (Phosphor changes all classes to implement this interface), and use the get and set methods. +To get or set the taint tag of an object, first cast that object to the interface TaintedWithObjTag (Phosphor changes +all classes to implement this interface), and use the get and set methods. + +You can determine if a variable is derived from a particular tainted source by examining the labels on that +variable's `Taint` object. -You can determine if a variable is derived from a particular tainted source by examining the labels on that variable's `Taint` object. +You *can* detaint variables with Phosphor - to do so, simply use the `MultiTainter` interface to set the taint on a +value to `0` (or `null`). -You *can* detaint variables with Phosphor - to do so, simply use the `MultiTainter` interface to set the taint on a value to `0` (or `null`). +## Notes on control tracking -Building ------- -Phosphor is a maven project. You can generate the jar with a simple `mvn package`. You can run the tests with `mvn verify` (which also generates the jar). Phosphor requires Java 9+ to build, but can be used on Java 8 VMs. If you are making changes to Phosphor and running the tests, you will want to make sure that Phosphor regenerates the instrumented JRE between test runs (because you are changing the instrumentation process). To do so, simply do `mvn clean verify` instead. +Please note that the control tracking functionality can impose SIGNIFICANT overhead (we've observed > 10x slowdown) +depending on the structure of the code you are instrumenting and the amount of tainted data flowing around. This is +incredibly un-optimized at this point. This also can make it difficult to apply Phosphor with control tracking to very +large methods (since it causes them to grow beyond the maximum size permitted). Nonetheless, we have had great success +applying it in various projects --- it works fine on the JDK (perhaps a few internal classes will be too large, but they +were not needed in our workloads) and on projects like Tomcat. There are quite a few paths to improving this +functionality. If you are interested in helping, please contact us. -Notes on control tracking ------ -Please note that the control tracking functionality can impose SIGNIFICANT overhead (we've observed > 10x slowdown) depending on the structure of the code you are instrumenting and the amount of tainted data flowing around. This is incredibly un-optimized at this point. This also can make it difficult to apply Phosphor with control tracking to very large methods (since it causes them to grow beyond the maximum size permitted). Nonetheless, we have had great success applying it in various projects --- it works fine on the JDK (perhaps a few internal classes will be too large, but they were not needed in our workloads) and on projects like Tomcat. There are quite a few paths to improving this functionality. If you are interested in helping, please contact us. +## Contact -Questions, concerns, comments ----- -Please email [Jonathan Bell](mailto:bellj@gmu.edu) with any feedback. This project is still under heavy development, and we are working on many extensions, and would very much welcome any feedback. +Please email [Jonathan Bell](mailto:bellj@gmu.edu) with any feedback or questions. +This project is still under development and we welcome any feedback. + +## License -License -------- This software is released under the MIT license. Copyright (c) 2013, by The Trustees of Columbia University in the City of New York. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +## Acknowledgements -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Phosphor makes use of the following libraries: -Acknowledgements --------- -This project makes use of the following libraries: -* [ASM](http://asm.ow2.org/license.html), (c) 2000-2011 INRIA, France Telecom, [license](http://asm.ow2.org/license.html) -* [Apache Harmony](https://harmony.apache.org), (c) The Apache Software Foundation, [license](http://www.apache.org/licenses/LICENSE-2.0) +* [ASM](http://asm.ow2.org/license.html), (c) 2000-2011 INRIA, France + Telecom, [license](http://asm.ow2.org/license.html) +* [Apache Harmony](https://harmony.apache.org), (c) The Apache Software + Foundation, [license](http://www.apache.org/licenses/LICENSE-2.0) -Phosphor's performance tuning is made possible by [JProfiler, the java profiler](https://www.ej-technologies.com/products/jprofiler/overview.html). +Phosphor's performance tuning is made possible +by [JProfiler, the java profiler](https://www.ej-technologies.com/products/jprofiler/overview.html). -The authors of this software are Katherine Hough, [Jonathan Bell](http://jonbell.net) and [Gail Kaiser](http://www.cs.columbia.edu/~kaiser/). Jonathan Bell and Katherine Hough are funded in part by NSF CCF-1763822 and CCF-1844880. Gail Kaiser directs the [Programming Systems Laboratory](http://www.psl.cs.columbia.edu/), funded in part by NSF CCF-1161079, NSF CNS-0905246, and NIH U54 CA121852. +The authors of this software are Katherine Hough, [Jonathan Bell](http://jonbell.net) +and [Gail Kaiser](http://www.cs.columbia.edu/~kaiser/). Jonathan Bell and Katherine Hough are funded in part by NSF +CCF-1763822 and CCF-1844880. Gail Kaiser directs the [Programming Systems Laboratory](http://www.psl.cs.columbia.edu/), +funded in part by NSF CCF-1161079, NSF CNS-0905246, and NIH U54 CA121852. From 1a8ec9e809639a8b11e5ba675f4939f8884b6f29 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 13:10:48 -0500 Subject: [PATCH 06/12] * Started working on Jar driver --- Phosphor/pom.xml | 36 ++----- .../columbia/cs/psl/phosphor/OptionsUtil.java | 88 ----------------- .../cs/psl/phosphor/PhosphorOption.java | 95 +++++++++++++++---- .../src/main/resources/META-INF/MANIFEST.MF | 5 - .../swe/phosphor/instrument/Instrumenter.java | 46 ++++----- .../instrument/PhosphorInstrumentation.java | 23 ++++- .../instrument/PhosphorInstrumenter.java | 25 +++-- pom.xml | 2 +- 8 files changed, 144 insertions(+), 176 deletions(-) delete mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java delete mode 100644 Phosphor/src/main/resources/META-INF/MANIFEST.MF diff --git a/Phosphor/pom.xml b/Phosphor/pom.xml index 9d0ef5706..db28b1a93 100644 --- a/Phosphor/pom.xml +++ b/Phosphor/pom.xml @@ -17,37 +17,19 @@ maven-jar-plugin - src/main/resources/META-INF/MANIFEST.MF + + + edu.columbia.cs.psl.phosphor.Java8PreMain + + true + - - java/ - sun/ - + true - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.1 - - none - true - 8 - sun.*:java.* - - - - attach-javadoc - - jar - - - - org.codehaus.mojo exec-maven-plugin - 1.6.0 generate-stubs @@ -95,7 +77,8 @@ - edu/columbia/cs/psl/phosphor/org/objectweb/asm/tree/FrameNode.class + + edu/columbia/cs/psl/phosphor/org/objectweb/asm/tree/FrameNode.class @@ -109,6 +92,7 @@ + true true true diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java deleted file mode 100644 index 0bd356470..000000000 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/OptionsUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -package edu.columbia.cs.psl.phosphor; - -import edu.columbia.cs.psl.phosphor.struct.SinglyLinkedList; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; - -import java.util.*; - -public class OptionsUtil { - /** - * Creates a standardized copy of the specified properties where each Phosphor option without an argument is mapped - * to either true or not present in the properties, each Phosphor option with an argument is either mapped to a - * non-null, non-empty string or not present in properties, and no other keys are present in the properties. - * - * @param properties properties to be standardized - * @return a standardized copy of properties - */ - private static Properties standardize(Properties properties) { - Properties result = new Properties(); - Map phosphorOptionMap = createOptionMap(); - for (String key : properties.stringPropertyNames()) { - String value = properties.getProperty(key); - if (phosphorOptionMap.containsKey(key)) { - Option option = phosphorOptionMap.get(key); - if (option.hasArg() && value != null && !value.isEmpty()) { - result.setProperty(option.getOpt(), value); - } else if (!option.hasArg() && (value == null || value.isEmpty() || "true".equalsIgnoreCase(value))) { - result.setProperty(option.getOpt(), "true"); - } - } else { - throw new IllegalArgumentException("Unknown option: " + key); - } - } - return result; - } - - /** - * @return a mapping from the names of configuration options available in Phosphor to an instance of - * org.apache.commons.cli.Option that represents that configuration option - */ - private static Map createOptionMap() { - Map map = new HashMap<>(); - Options options = PhosphorOption.createOptions(false); - for (Option option : options.getOptions()) { - map.put(option.getOpt(), option); - if (option.hasLongOpt()) { - map.put(option.getLongOpt(), option); - } - } - return map; - } - - public static String[] createPhosphorMainArguments(Properties properties) { - properties = standardize(properties); - SinglyLinkedList arguments = new SinglyLinkedList<>(); - Set propNames = properties.stringPropertyNames(); - for (String propName : propNames) { - arguments.addLast("-" + propName); - if (!"true".equals(properties.getProperty(propName))) { - arguments.addLast(properties.getProperty(propName)); - } - } - arguments.addLast("temp/"); - arguments.addLast("temp2/"); - return arguments.toArray(new String[0]); - } - - public static Set> getConfigurationClasses(Properties properties) { - Set> classes = new HashSet<>(); - Options options = PhosphorOption.createOptions(false); - for (Option option : options.getOptions()) { - if (option.getType().equals(Class.class)) { - String key = option.getOpt(); - if (properties.containsKey(key)) { - try { - Class clazz = Class.forName(properties.getProperty(key)); - classes.add(clazz); - } catch (ReflectiveOperationException e) { - String message = - String.format("Failed to create %s class: %s", key, properties.getProperty(key)); - throw new IllegalArgumentException(message, e); - } - } - } - } - return classes; - } -} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java index 3087b1809..522fd76fa 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java @@ -7,9 +7,10 @@ import org.apache.commons.cli.*; import org.objectweb.asm.ClassVisitor; +import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.util.EnumMap; +import java.util.*; public enum PhosphorOption { @@ -369,42 +370,94 @@ public static Options createOptions(boolean forRuntimeInst) { } } } - for(OptionGroup group : groupMap.values()) { + for (OptionGroup group : groupMap.values()) { options.addOptionGroup(group); } return options; } - public static CommandLine configure(boolean forRuntimeInst, String[] args) { - CommandLineParser parser = new DefaultParser(); - Options options = createOptions(forRuntimeInst); + public static CommandLine configure(boolean isRuntime, String[] args) { + String commandSynopsis = "java -jar phosphor.jar [OPTIONS] "; + Options options = createOptions(isRuntime); CommandLine line; try { - line = parser.parse(options, args); - } catch(org.apache.commons.cli.ParseException exp) { - if(forRuntimeInst) { - System.err.println(exp.getMessage()); + line = new DefaultParser().parse(options, args); + } catch (ParseException e) { + if (isRuntime) { + System.err.println(e.getMessage()); return null; } - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("java -jar phosphor.jar [OPTIONS] [input] [output]", options); - System.err.println(exp.getMessage()); - if(exp.getMessage().contains("-multiTaint")) { - System.err.println("Note: the -multiTaint option has been removed, and is now enabled by default (int tags no longer exist)"); - } - return null; + new HelpFormatter().printHelp(commandSynopsis, options); + throw new IllegalArgumentException(e); } - if(!forRuntimeInst && (line.hasOption("help") || line.getArgs().length != 2)) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("java -jar phosphor.jar [OPTIONS] [input] [output]", options); + if (!isRuntime && line.hasOption("help")) { + new HelpFormatter().printHelp(commandSynopsis, options); return null; } - for(PhosphorOption phosphorOption : values()) { - phosphorOption.configure(forRuntimeInst, line.hasOption(phosphorOption.optionName), line); + if (!isRuntime && line.getArgs().length != 2) { + new HelpFormatter().printHelp(commandSynopsis, options); + throw new IllegalArgumentException("Missing command line arguments"); + } + for (PhosphorOption phosphorOption : values()) { + phosphorOption.configure(isRuntime, line.hasOption(phosphorOption.optionName), line); } return line; } + public static Set> getClassOptionValues(CommandLine line) { + // This method should not be called from instrumented JVMs. + // Therefore, the use of JCL classes is acceptable. + Set> classes = new HashSet<>(); + for (Option option : line.getOptions()) { + if (option.getType().equals(Class.class)) { + if (line.hasOption(option.getOpt())) { + try { + classes.add((Class) line.getParsedOptionValue(option.getOpt())); + } catch (ParseException e) { + throw new IllegalArgumentException( + "Failed to process " + option.getOpt() + ": " + line.getOptionValue(option.getOpt())); + } + } + } + } + return classes; + } + + public static CommandLine configure(Properties properties, File source, File destination) { + // This method should not be called from instrumented JVMs. + // Therefore, the use of JCL classes is acceptable. + List arguments = new LinkedList<>(); + for (String key : properties.stringPropertyNames()) { + String value = properties.getProperty(key); + if (value == null || value.isEmpty() || value.equals("true")) { + arguments.add("-" + key); + } else if (!value.equals("false")) { + arguments.add("-" + key); + arguments.add(value); + } + } + arguments.add(source.getAbsolutePath()); + arguments.add(destination.getAbsolutePath()); + return configure(false, arguments.toArray(new String[0])); + } + + public static Properties toProperties(CommandLine line) { + // This method should not be called from instrumented JVMs. + // Therefore, the use of JCL classes is acceptable. + Properties properties = new Properties(); + for (Option option : line.getOptions()) { + String key = option.getOpt(); + if (line.hasOption(key)) { + String value = line.getOptionValue(key); + if (value == null) { + value = "true"; + } + properties.put(key, value); + } + } + return properties; + } + private enum PhosphorOptionGroup { GENERAL, CONTROL_PROPAGATION } diff --git a/Phosphor/src/main/resources/META-INF/MANIFEST.MF b/Phosphor/src/main/resources/META-INF/MANIFEST.MF deleted file mode 100644 index f8fd3bc8e..000000000 --- a/Phosphor/src/main/resources/META-INF/MANIFEST.MF +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -Created-By: 1.6.0_06 (Sun Microsystems Inc.) -Main-Class: edu.columbia.cs.psl.phosphor.Instrumenter -Premain-Class: edu.columbia.cs.psl.phosphor.Java8PreMain -Can-Retransform-Classes: true diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java index 87e645525..cb7158a3e 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java @@ -27,14 +27,14 @@ public Instrumenter(Instrumentation instrumentation, boolean verbose) { this.verbose = verbose; } - public void process(File source, File target) throws IOException, InterruptedException, ExecutionException { + public void process(File source, File destination) throws IOException, InterruptedException, ExecutionException { if (!source.exists()) { throw new IllegalArgumentException("Source file not found: " + source); } else if (!source.isDirectory() && !isClass(source.getName()) && !isArchive(source.getName())) { throw new IllegalArgumentException("Unknown source file type: " + source); } Queue> futures = new LinkedList<>(); - processFile(futures, source, target); + processFile(futures, source, destination); while (!futures.isEmpty()) { futures.poll().get(); } @@ -51,9 +51,9 @@ public void process(File source, File target) throws IOException, InterruptedExc } } - private void instrumentClass(File source, File target) { + private void instrumentClass(File source, File destination) { try (InputStream input = Files.newInputStream(source.toPath()); - OutputStream output = Files.newOutputStream(target.toPath())) { + OutputStream output = Files.newOutputStream(destination.toPath())) { instrumentClass(InstrumentUtil.readAllBytes(input), output); } catch (Throwable t) { errors.add(t); @@ -69,27 +69,27 @@ private void instrumentClass(byte[] classFileBuffer, OutputStream output) throws } } - private void processFile(Collection> futures, File source, File target) + private void processFile(Collection> futures, File source, File destination) throws IOException, InterruptedException { if (source.isDirectory()) { - InstrumentUtil.ensureDirectory(target); + InstrumentUtil.ensureDirectory(destination); for (File child : Objects.requireNonNull(source.listFiles())) { - processFile(futures, child, new File(target, child.getName())); + processFile(futures, child, new File(destination, child.getName())); } } else if (isClass(source.getName())) { - futures.add(executor.submit(() -> instrumentClass(source, target), null)); + futures.add(executor.submit(() -> instrumentClass(source, destination), null)); } else if (isArchive(source.getName())) { - processZip(Files.newInputStream(source.toPath()), Files.newOutputStream(target.toPath())); + processZip(Files.newInputStream(source.toPath()), Files.newOutputStream(destination.toPath())); } else { - if (copy(source, target)) { - if (source.canExecute() && !target.setExecutable(true)) { - errors.add(new IOException("Failed to set execute permission for: " + target)); + if (copy(source, destination)) { + if (source.canExecute() && !destination.setExecutable(true)) { + errors.add(new IOException("Failed to set execute permission for: " + destination)); } - if (source.canRead() && !target.setReadable(true)) { - errors.add(new IOException("Failed to set read permission for: " + target)); + if (source.canRead() && !destination.setReadable(true)) { + errors.add(new IOException("Failed to set read permission for: " + destination)); } - if (source.canWrite() && !target.setWritable(true)) { - errors.add(new IOException("Failed to set write permission for: " + target)); + if (source.canWrite() && !destination.setWritable(true)) { + errors.add(new IOException("Failed to set write permission for: " + destination)); } } } @@ -144,9 +144,9 @@ private void writeZipResults(OutputStream out, List> futures) } } - private boolean copy(File source, File target) { + private boolean copy(File source, File destination) { try (InputStream in = Files.newInputStream(source.toPath()); - OutputStream out = Files.newOutputStream(target.toPath())) { + OutputStream out = Files.newOutputStream(destination.toPath())) { InstrumentUtil.copy(in, out); return true; } catch (IOException e) { @@ -193,14 +193,14 @@ public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedE public static long instrument( File source, - File target, + File destination, Properties options, Instrumentation instrumentation, boolean verbose, String modules) throws IOException { - if (target.exists()) { - throw new IllegalArgumentException("Target location for instrumentation already exists."); + if (destination.exists()) { + throw new IllegalArgumentException("Destination location for instrumentation already exists."); } if (!source.exists()) { throw new IllegalArgumentException("Source location not found: " + source); @@ -208,9 +208,9 @@ public static long instrument( long startTime = System.currentTimeMillis(); try { if (InstrumentUtil.isModularJvm(source)) { - JLinkInvoker.invoke(source, target, instrumentation, options, modules); + JLinkInvoker.invoke(source, destination, instrumentation, options, modules); } else { - new Instrumenter(instrumentation, verbose).process(source, target); + new Instrumenter(instrumentation, verbose).process(source, destination); } return System.currentTimeMillis() - startTime; } catch (IOException | InterruptedException | ExecutionException e) { diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java index e7b0de35b..6494fe1e2 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java @@ -1,7 +1,11 @@ package edu.gmu.swe.phosphor.instrument; -import edu.columbia.cs.psl.phosphor.*; +import edu.columbia.cs.psl.phosphor.Configuration; +import edu.columbia.cs.psl.phosphor.PhosphorOption; +import edu.columbia.cs.psl.phosphor.PhosphorPatcher; +import edu.columbia.cs.psl.phosphor.PreMain; import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; +import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.CommandLine; import java.io.File; import java.io.IOException; @@ -25,16 +29,25 @@ public class PhosphorInstrumentation implements Instrumentation { @Override public void configure(File source, Properties options) throws IOException { - options.put("java8", String.valueOf(!InstrumentUtil.isModularJvm(source))); - String[] arguments = OptionsUtil.createPhosphorMainArguments(options); + options.remove("help"); + CommandLine line = PhosphorOption.configure(options, source, new File("temp/")); + assert line != null; + initialize(line); + } + + public void initialize(CommandLine line) throws IOException { + File source = new File(line.getArgs()[0]); + Set> configurationClasses = PhosphorOption.getClassOptionValues(line); + if (InstrumentUtil.isJavaHome(source)) { + Configuration.IS_JAVA_8 = !InstrumentUtil.isModularJvm(source); + } setUpClassLoader(source); - PhosphorOption.configure(false, arguments); Configuration.init(); TaintTrackingClassVisitor.IS_RUNTIME_INST = false; transformer = new PreMain.PCLoggingTransformer(); classPathElements = new HashSet<>(); classPathElements.add(InstrumentUtil.getClassPathElement(PreMain.class)); - OptionsUtil.getConfigurationClasses(options).stream() + configurationClasses.stream() .map(InstrumentUtil::getClassPathElement) .forEach(classPathElements::add); } diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java index 1f4715bc7..d59e92dc0 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java +++ b/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java @@ -1,5 +1,9 @@ package edu.gmu.swe.phosphor.instrument; +import edu.columbia.cs.psl.phosphor.Configuration; +import edu.columbia.cs.psl.phosphor.PhosphorOption; +import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.CommandLine; + import java.io.File; import java.io.IOException; @@ -9,13 +13,20 @@ private PhosphorInstrumenter() { } public static void main(String[] args) throws IOException { - // TODO + PhosphorInstrumentation instrumentation = new PhosphorInstrumentation(); + CommandLine line = PhosphorOption.configure(false, args); + if(line == null) { + // The "help" option was specified + return; + } File source = new File(args[0]); - File target = new File(args[1]); - boolean verbose = Boolean.parseBoolean(args[2]); - String modules = args[3]; - System.out.printf("Instrumenting %s to %s%n", source, target); - long elapsedTime = Instrumenter.instrument(source, target, null, null, verbose, modules); - System.out.printf("Finished generation after %dms%n", elapsedTime); + File destination = new File(args[1]); + instrumentation.initialize(line); + boolean verbose = !Configuration.QUIET_MODE; + String modules = line.hasOption("jvmModules") ? line.getOptionValue("jvmModules") : "ALL-MODULE-PATH"; + System.out.printf("Instrumenting %s to %s%n", source, destination); + long elapsedTime = Instrumenter.instrument(source, destination, PhosphorOption.toProperties(line), + instrumentation, verbose, modules); + System.out.printf("Finished instrumentation after %dms%n", elapsedTime); } } diff --git a/pom.xml b/pom.xml index d0b6e72e4..15275c612 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ MIT License - http://www.opensource.org/licenses/mit-license.php + https://www.opensource.org/licenses/mit-license.php repo From d380406c4b69754a47bf73bf433ff19277bb586c Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 15:50:05 -0500 Subject: [PATCH 07/12] * Combined Java 8 and Java 9+ agents into a single agent * Renamed gmu packages for consistency * Removed unnecessary modules * Renamed phosphor-instrument-jigsaw module to phosphor-driver --- Phosphor/pom.xml | 66 ++-- .../psl/phosphor/BasicSourceSinkManager.java | 2 +- .../cs/psl/phosphor/HackyClassWriter.java | 6 +- ...elper.java => InstrumentationAdaptor.java} | 11 +- .../cs/psl/phosphor/Java8PreMain.java | 81 ---- .../cs/psl/phosphor/PCLoggingTransformer.java | 300 ++++++++++++++ .../columbia/cs/psl/phosphor/Phosphor.java | 68 ++++ .../psl/phosphor/PhosphorBaseTransformer.java | 32 +- .../edu/columbia/cs/psl/phosphor/PreMain.java | 370 ------------------ .../psl/phosphor/SourceSinkTransformer.java | 28 +- .../cs/psl/phosphor/agent/PhosphorAgent.java | 100 +++++ .../ConfigurationEmbeddingMV.java | 3 +- .../instrumenter/TaintMethodRecord.java | 11 +- .../cs/psl/phosphor/runtime/StringUtils.java | 6 +- .../RuntimeSunMiscUnsafePropagator.java | 4 +- .../cs/psl/phosphor/InstrumenterTest.java | 8 +- integration-tests/pom.xml | 8 +- integration-tests/runDacapo.sh | 5 +- .../psl/test/phosphor/BasePhosphorTest.java | 6 +- .../runtime/ReflectionImplicitITCase.java | 5 +- phosphor-distribution/pom.xml | 51 --- phosphor-distribution/src/assembly/bin.xml | 23 -- .../pom.xml | 15 +- .../phosphor/driver}/DeletingFileVisitor.java | 2 +- .../driver}/InstrumentJLinkPlugin.java | 2 +- .../psl/phosphor/driver}/InstrumentUtil.java | 2 +- .../psl/phosphor/driver}/Instrumentation.java | 2 +- .../cs/psl/phosphor/driver}/Instrumenter.java | 2 +- .../cs/psl/phosphor/driver}/JLinkInvoker.java | 2 +- .../driver}/JLinkRegistrationAgent.java | 4 +- .../cs/psl/phosphor/driver}/Packer.java | 2 +- .../cs/psl/phosphor/driver}/Patcher.java | 2 +- .../driver}/PhosphorInstrumentation.java | 21 +- .../driver}/PhosphorInstrumenter.java | 2 +- .../src/main/java/module-info.java | 6 + .../src/main/java/module-info.java | 5 - phosphor-instrument-maven-plugin/pom.xml | 2 +- .../psl/phosphor/plugin}/InstrumentMojo.java | 10 +- phosphor-jigsaw-javaagent/pom.xml | 93 ----- .../jigsaw/phosphor/javaagent/PreMain.java | 75 ---- .../src/main/java/module-info.java | 4 - pom.xml | 4 +- 42 files changed, 607 insertions(+), 844 deletions(-) rename Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/{InstrumentationHelper.java => InstrumentationAdaptor.java} (50%) delete mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java create mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PCLoggingTransformer.java create mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java delete mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PreMain.java create mode 100644 Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/agent/PhosphorAgent.java delete mode 100644 phosphor-distribution/pom.xml delete mode 100644 phosphor-distribution/src/assembly/bin.xml rename {phosphor-instrument-jigsaw => phosphor-driver}/pom.xml (90%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/DeletingFileVisitor.java (96%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/InstrumentJLinkPlugin.java (98%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/InstrumentUtil.java (98%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/Instrumentation.java (97%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/Instrumenter.java (99%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/JLinkInvoker.java (98%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/JLinkRegistrationAgent.java (99%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/Packer.java (99%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/Patcher.java (75%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/PhosphorInstrumentation.java (84%) rename {phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument => phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver}/PhosphorInstrumenter.java (96%) create mode 100644 phosphor-driver/src/main/java/module-info.java delete mode 100644 phosphor-instrument-jigsaw/src/main/java/module-info.java rename phosphor-instrument-maven-plugin/src/main/java/edu/{gmu/swe/phosphor/instrument => columbia/cs/psl/phosphor/plugin}/InstrumentMojo.java (96%) delete mode 100644 phosphor-jigsaw-javaagent/pom.xml delete mode 100644 phosphor-jigsaw-javaagent/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/javaagent/PreMain.java delete mode 100644 phosphor-jigsaw-javaagent/src/main/java/module-info.java diff --git a/Phosphor/pom.xml b/Phosphor/pom.xml index db28b1a93..5a648514e 100644 --- a/Phosphor/pom.xml +++ b/Phosphor/pom.xml @@ -10,6 +10,38 @@ phosphor-parent 0.1.0-SNAPSHOT + + + commons-cli + commons-cli + 1.4 + + + org.ow2.asm + asm + 9.6 + + + org.ow2.asm + asm-util + 9.6 + + + org.ow2.asm + asm-tree + 9.6 + + + org.ow2.asm + asm-analysis + 9.6 + + + org.ow2.asm + asm-commons + 9.6 + + @@ -19,7 +51,7 @@ - edu.columbia.cs.psl.phosphor.Java8PreMain + edu.columbia.cs.psl.phosphor.agent.PhosphorAgent true @@ -109,36 +141,4 @@ - - - commons-cli - commons-cli - 1.4 - - - org.ow2.asm - asm - 9.6 - - - org.ow2.asm - asm-util - 9.6 - - - org.ow2.asm - asm-tree - 9.6 - - - org.ow2.asm - asm-analysis - 9.6 - - - org.ow2.asm - asm-commons - 9.6 - - diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/BasicSourceSinkManager.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/BasicSourceSinkManager.java index 4b5cb0a18..9c849fb5a 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/BasicSourceSinkManager.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/BasicSourceSinkManager.java @@ -232,7 +232,7 @@ public static synchronized java.util.LinkedList replaceAutoTaintMethods( try { if (classMap.containsKey(className)) { for (Class clazz : classMap.get(className)) { - PreMain.getInstrumentationHelper().retransformClasses(clazz); + Phosphor.getInstrumentation().retransformClasses(clazz); } } } catch (NonModifiableClassException e) { diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/HackyClassWriter.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/HackyClassWriter.java index 0bc97b386..0d9d00962 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/HackyClassWriter.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/HackyClassWriter.java @@ -11,14 +11,14 @@ public HackyClassWriter(ClassReader classReader, int flags) { @Override protected String getCommonSuperClass(String type1, String type2) { - if (PreMain.RUNTIME_INST) { + if (Phosphor.RUNTIME_INST) { return "java/lang/Object"; } Class clazz1; Class clazz2; try { - clazz1 = Class.forName(type1.replace('/', '.'), false, PreMain.bigLoader); - clazz2 = Class.forName(type2.replace('/', '.'), false, PreMain.bigLoader); + clazz1 = Class.forName(type1.replace('/', '.'), false, Phosphor.bigLoader); + clazz2 = Class.forName(type2.replace('/', '.'), false, Phosphor.bigLoader); } catch (Throwable t) { return "java/lang/Object"; } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationHelper.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationAdaptor.java similarity index 50% rename from Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationHelper.java rename to Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationAdaptor.java index 317d8ac4b..ddc3ff3c5 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationHelper.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/InstrumentationAdaptor.java @@ -2,11 +2,12 @@ import edu.columbia.cs.psl.phosphor.runtime.NonModifiableClassException; -public interface InstrumentationHelper { - void addTransformer(PhosphorBaseTransformer classSupertypeReadingTransformer); - void addTransformer(PhosphorBaseTransformer classSupertypeReadingTransformer, boolean canRedefineClasses); - - void retransformClasses(Class clazz) throws NonModifiableClassException; +public interface InstrumentationAdaptor { + void addTransformer(PhosphorBaseTransformer transformer); Class[] getAllLoadedClasses(); + + void addTransformer(PhosphorBaseTransformer transformer, boolean canRedefineClasses); + + void retransformClasses(Class clazz) throws NonModifiableClassException; } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java deleted file mode 100644 index bf8e02b8d..000000000 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Java8PreMain.java +++ /dev/null @@ -1,81 +0,0 @@ -package edu.columbia.cs.psl.phosphor; - - -import edu.columbia.cs.psl.phosphor.runtime.NonModifiableClassException; -import edu.columbia.cs.psl.phosphor.runtime.PhosphorStackFrame; - -import java.lang.instrument.ClassFileTransformer; -import java.lang.instrument.IllegalClassFormatException; -import java.lang.instrument.Instrumentation; -import java.lang.instrument.UnmodifiableClassException; -import java.security.ProtectionDomain; - -import static edu.columbia.cs.psl.phosphor.PhosphorBaseTransformer.INITED; - -/** - * We use a premain in a separate jar for Java9+, in the phosphor-jigsaw-javaagent package - */ -public class Java8PreMain { - static class PhosphorTransformerBridge implements ClassFileTransformer { - private PhosphorBaseTransformer transformer; - - public PhosphorTransformerBridge(PhosphorBaseTransformer transformer) { - this.transformer = transformer; - } - - @Override - public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException { - return transform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer, null); - } - - public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer, PhosphorStackFrame phosphorStackFrame) throws IllegalClassFormatException { - try { - if (!INITED) { - Configuration.init(); - INITED = true; - } - return transformer.signalAndTransform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer, false); - } catch (Throwable t) { - t.printStackTrace(); - throw t; - } - } - } - - public static void premain(String args, Instrumentation instr, PhosphorStackFrame phosphorStackFrame) { - premain(args, instr); - } - - public static void premain(String args, Instrumentation instr) { - if (args == null) { - args = "java8"; - } else { - args += ",java8"; - } - edu.columbia.cs.psl.phosphor.PreMain.premain(args, new InstrumentationHelper() { - @Override - public void addTransformer(final PhosphorBaseTransformer transformer) { - instr.addTransformer(new PhosphorTransformerBridge(transformer)); - } - - @Override - public Class[] getAllLoadedClasses() { - return instr.getAllLoadedClasses(); - } - - @Override - public void addTransformer(PhosphorBaseTransformer transformer, boolean canRedefineClasses) { - instr.addTransformer(new PhosphorTransformerBridge(transformer), canRedefineClasses); - } - - @Override - public void retransformClasses(Class clazz) throws NonModifiableClassException { - try { - instr.retransformClasses(clazz); - } catch (UnmodifiableClassException e) { - throw new NonModifiableClassException(e); - } - } - }); - } -} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PCLoggingTransformer.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PCLoggingTransformer.java new file mode 100644 index 000000000..fdd4d67a2 --- /dev/null +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PCLoggingTransformer.java @@ -0,0 +1,300 @@ +package edu.columbia.cs.psl.phosphor; + +import edu.columbia.cs.psl.phosphor.instrumenter.*; +import edu.columbia.cs.psl.phosphor.instrumenter.asm.OffsetPreservingClassReader; +import edu.columbia.cs.psl.phosphor.org.objectweb.asm.commons.OurSerialVersionUIDAdder; +import edu.columbia.cs.psl.phosphor.runtime.StringUtils; +import edu.columbia.cs.psl.phosphor.runtime.TaintInstrumented; +import edu.columbia.cs.psl.phosphor.struct.TaintedWithObjTag; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.HashSet; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.LinkedList; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.List; +import edu.columbia.cs.psl.phosphor.struct.harmony.util.Set; +import org.objectweb.asm.*; +import org.objectweb.asm.tree.AnnotationNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.FieldNode; +import org.objectweb.asm.tree.MethodNode; +import org.objectweb.asm.util.CheckClassAdapter; +import org.objectweb.asm.util.TraceClassVisitor; + +import java.io.*; +import java.lang.reflect.Constructor; +import java.security.ProtectionDomain; + +public final class PCLoggingTransformer extends PhosphorBaseTransformer { + + public PCLoggingTransformer() { + TaintUtils.VERIFY_CLASS_GENERATION = System.getProperty("phosphor.verify") != null; + } + + @Override + public byte[] transform(ClassLoader loader, final String className2, Class classBeingRedefined, + ProtectionDomain protectionDomain, byte[] classfileBuffer, boolean isAnonymousClassDefinition) { + ClassReader cr = (Configuration.READ_AND_SAVE_BCI ? new OffsetPreservingClassReader(classfileBuffer) + : new ClassReader(classfileBuffer)); + String className = cr.getClassName(); + if (Instrumenter.isIgnoredClass(className)) { + switch (className) { + case "java/lang/Boolean": + case "java/lang/Byte": + case "java/lang/Character": + case "java/lang/Short": + return processBoolean(classfileBuffer); + } + return classfileBuffer; + } + Configuration.taintTagFactory.instrumentationStarting(className); + try { + ClassNode cn = new ClassNode(); + cr.accept(cn, ClassReader.SKIP_CODE); + boolean upgradeVersion = false; + if (className.equals("org/jruby/parser/Ruby20YyTables")) { + cn.version = 51; + upgradeVersion = true; + } + if (cn.invisibleAnnotations != null) { + for (Object o : cn.invisibleAnnotations) { + AnnotationNode an = (AnnotationNode) o; + if (an.desc.equals(Type.getDescriptor(TaintInstrumented.class))) { + return classfileBuffer; + } + } + } + if (cn.interfaces != null) { + for (Object s : cn.interfaces) { + if (s.equals(Type.getInternalName(TaintedWithObjTag.class))) { + return classfileBuffer; + } + } + } + for (Object mn : cn.methods) { + if (((MethodNode) mn).name.equals("getPHOSPHOR_TAG")) { + return classfileBuffer; + } + } + if (Configuration.CACHE != null) { + byte[] cachedClass = Configuration.CACHE.load(className, classfileBuffer); + if (cachedClass != null) { + return cachedClass; + } + } + if (Phosphor.DEBUG) { + try { + File debugDir = new File("debug-preinst"); + if (!debugDir.exists()) { + debugDir.mkdir(); + } + File f = new File("debug-preinst/" + className.replace("/", ".") + ".class"); + FileOutputStream fos = new FileOutputStream(f); + fos.write(classfileBuffer); + fos.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + boolean isiFace = (cn.access & Opcodes.ACC_INTERFACE) != 0; + List fields = new LinkedList<>(); + for (FieldNode node : cn.fields) { + fields.add(node); + } + boolean skipFrames = LegacyClassFixer.shouldFixFrames(cn, className, cr); + if (skipFrames) { + // This class is old enough to not guarantee frames. + // Generate new frames for analysis reasons, then make sure to not emit ANY frames. + cr = LegacyClassFixer.fix(cr); + } + try { + byte[] instrumentedBytes = instrumentWithRetry(cr, classfileBuffer, isiFace, className, skipFrames, + upgradeVersion, fields, null, false, isAnonymousClassDefinition); + if (Phosphor.DEBUG) { + File f = new File("debug/" + className + ".class"); + f.getParentFile().mkdirs(); + FileOutputStream fos = new FileOutputStream(f); + fos.write(instrumentedBytes); + fos.close(); + } + if (Configuration.CACHE != null) { + Configuration.CACHE.store(className, classfileBuffer, instrumentedBytes); + } + return instrumentedBytes; + } catch (Throwable ex) { + ex.printStackTrace(); + throw new IllegalStateException(ex); + } + } finally { + Configuration.taintTagFactory.instrumentationEnding(className); + } + } + + static byte[] instrumentWithRetry(ClassReader cr, byte[] classFileBuffer, boolean isiFace, String className, + boolean skipFrames, boolean upgradeVersion, List fields, + Set methodsToReduceSizeOf, boolean traceClass, boolean isAnonymousClassDefinition) { + TraceClassVisitor debugTracer = null; + try { + try { + ClassWriter cw = new HackyClassWriter(null, ClassWriter.COMPUTE_MAXS); + ClassVisitor _cv = cw; + if (traceClass) { + System.out.println("Saving " + className + " to debug-preinst/"); + File f = new File("debug-preinst/" + className.replace("/", ".") + + ".class"); + if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { + System.err.println("Failed to make debug directory: " + f); + } else { + try { + FileOutputStream fos = new FileOutputStream(f); + fos.write(classFileBuffer); + fos.close(); + } catch (Exception ex2) { + ex2.printStackTrace(); + } + } + debugTracer = new TraceClassVisitor(null, null); + _cv = debugTracer; + } + if (Configuration.POST_CLASS_VISITOR != null) { + try { + Constructor extra = Configuration.POST_CLASS_VISITOR + .getConstructor(ClassVisitor.class, Boolean.TYPE, byte[].class); + _cv = extra.newInstance(_cv, skipFrames, classFileBuffer); + } catch (Exception e) { + // + } + } + if (Configuration.extensionClassVisitor != null) { + Constructor extra = Configuration.extensionClassVisitor.getConstructor(ClassVisitor.class, Boolean.TYPE); + _cv = extra.newInstance(_cv, skipFrames); + } + if (Phosphor.DEBUG || TaintUtils.VERIFY_CLASS_GENERATION) { + _cv = new CheckClassAdapter(_cv, false); + } + if (SerializationFixingCV.isApplicable(className)) { + _cv = new SerializationFixingCV(_cv, className); + } + _cv = new ClinitRetransformClassVisitor(_cv); + boolean isGenerateConstructorAccessor = StringUtils.startsWith(className, "sun/reflect/GeneratedConstructor") + || StringUtils.startsWith(className, "jdk/internal/reflect/GeneratedConstructor"); // Calculating SVUID could trigger constructor generation causing stack overflow/infinite recursion + if (isiFace || isGenerateConstructorAccessor) { + _cv = new TaintTrackingClassVisitor(_cv, skipFrames, fields, methodsToReduceSizeOf, isAnonymousClassDefinition); + } else { + _cv = new OurSerialVersionUIDAdder(new TaintTrackingClassVisitor(_cv, skipFrames, fields, + methodsToReduceSizeOf, isAnonymousClassDefinition)); + } + if (EclipseCompilerCV.isEclipseCompilerClass(className)) { + _cv = new EclipseCompilerCV(_cv); + } + if (OgnlUtilCV.isOgnlUtilClass(className) && !Configuration.REENABLE_CACHES) { + _cv = new OgnlUtilCV(_cv); + } + if (JettyBufferUtilCV.isApplicable(className)) { + _cv = new JettyBufferUtilCV(_cv); + } + if (PowerMockUtilCV.isApplicable(className)) { + _cv = new PowerMockUtilCV(_cv); + } + if (Configuration.PRIOR_CLASS_VISITOR != null) { + try { + Constructor extra = Configuration.PRIOR_CLASS_VISITOR.getConstructor(ClassVisitor.class, Boolean.TYPE); + _cv = extra.newInstance(_cv, skipFrames); + } catch (Exception e) { + // + } + } + cr.accept(_cv, ClassReader.EXPAND_FRAMES); + byte[] instrumentedBytes = cw.toByteArray(); + if (!traceClass && (Phosphor.DEBUG || TaintUtils.VERIFY_CLASS_GENERATION)) { + ClassReader cr2 = new ClassReader(instrumentedBytes); + try { + cr2.accept(new CheckClassAdapter(new ClassWriter(0), true), ClassReader.EXPAND_FRAMES); + } catch (Throwable t) { + t.printStackTrace(); + File f = new File("debug-verify/" + className.replace("/", ".") + ".class"); + if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { + System.err.println("Failed to make debug directory: " + f); + } else { + try { + FileOutputStream fos = new FileOutputStream(f); + fos.write(instrumentedBytes); + fos.close(); + } catch (Exception ex2) { + ex2.printStackTrace(); + } + System.out.println("Saved broken class to " + f); + } + System.out.println("Saving uninstrumented " + className + " to debug-preinst/"); + f = new File("debug-preinst/" + className.replace("/", ".") + + ".class"); + if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { + System.err.println("Failed to make debug directory: " + f); + } else { + try { + FileOutputStream fos = new FileOutputStream(f); + fos.write(classFileBuffer); + fos.close(); + } catch (Exception ex2) { + ex2.printStackTrace(); + } + } + } + } + return instrumentedBytes; + } catch (MethodTooLargeException ex) { + if (methodsToReduceSizeOf == null) { + methodsToReduceSizeOf = new HashSet<>(); + } + if (!methodsToReduceSizeOf.add(ex.getMethodName() + ex.getDescriptor())) { + throw ex; //We already tried and failed to make this fit :( + } + return instrumentWithRetry(cr, classFileBuffer, isiFace, className, skipFrames, upgradeVersion, fields, methodsToReduceSizeOf, false, isAnonymousClassDefinition); + } + } catch (Throwable ex) { + Phosphor.INSTRUMENTATION_EXCEPTION_OCCURRED = true; + if (!traceClass) { + System.err.println("Exception occurred while instrumenting " + className + ":"); + ex.printStackTrace(); + instrumentWithRetry(cr, classFileBuffer, isiFace, className, skipFrames, upgradeVersion, fields, methodsToReduceSizeOf, true, isAnonymousClassDefinition); + return classFileBuffer; + } + ex.printStackTrace(); + System.err.println("method so far:"); + try { + PrintWriter pw = new PrintWriter(new FileWriter("lastClass.txt")); + debugTracer.p.print(pw); + pw.flush(); + } catch (IOException ex2) { + ex2.printStackTrace(); + } + return classFileBuffer; + } + } + + private static byte[] processBoolean(byte[] classFileBuffer) { + ClassReader cr = new ClassReader(classFileBuffer); + ClassNode cn = new ClassNode(Configuration.ASM_VERSION); + cr.accept(cn, 0); + boolean addField = true; + for (Object o : cn.fields) { + FieldNode fn = (FieldNode) o; + if (fn.name.equals("valueOf")) { + addField = false; + break; + } + } + for (Object o : cn.methods) { + MethodNode mn = (MethodNode) o; + if (mn.name.startsWith("toUpperCase") + || mn.name.startsWith("codePointAtImpl") + || mn.name.startsWith("codePointBeforeImpl")) { + mn.access = mn.access | Opcodes.ACC_PUBLIC; + } + } + if (addField) { + cn.fields.add(new FieldNode(Opcodes.ACC_PUBLIC, "valueOf", "Z", null, false)); + ClassWriter cw = new ClassWriter(0); + cn.accept(cw); + return cw.toByteArray(); + } + return classFileBuffer; + } +} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java new file mode 100644 index 000000000..0df15152e --- /dev/null +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java @@ -0,0 +1,68 @@ +package edu.columbia.cs.psl.phosphor; + +import edu.columbia.cs.psl.phosphor.instrumenter.InvokedViaInstrumentation; +import edu.columbia.cs.psl.phosphor.instrumenter.TaintMethodRecord; +import edu.columbia.cs.psl.phosphor.struct.SinglyLinkedList; + +public final class Phosphor { + public static boolean DEBUG = System.getProperty("phosphor.debug") != null; + public static boolean RUNTIME_INST = false; + public static boolean INSTRUMENTATION_EXCEPTION_OCCURRED = false; + public static ClassLoader bigLoader = Phosphor.class.getClassLoader(); + public static InstrumentationAdaptor instrumentation; + + private Phosphor() { + throw new AssertionError("Tried to instantiate static agent class: " + getClass()); + } + + public static void initialize(String agentArgs, InstrumentationAdaptor instrumentation) { + Phosphor.instrumentation = instrumentation; + instrumentation.addTransformer(new ClassSupertypeReadingTransformer()); + RUNTIME_INST = true; + PhosphorOption.configure(true, parseOptions(agentArgs)); + if (System.getProperty("phosphorCacheDirectory") != null) { + Configuration.CACHE = TransformationCache.getInstance(System.getProperty("phosphorCacheDirectory")); + } + if (Instrumenter.loader == null) { + Instrumenter.loader = bigLoader; + } + // Ensure that BasicSourceSinkManager and anything needed to call isSourceOrSinkOrTaintThrough gets initialized + BasicSourceSinkManager.loadTaintMethods(); + BasicSourceSinkManager.getInstance().isSourceOrSinkOrTaintThrough(Object.class); + instrumentation.addTransformer(new PCLoggingTransformer()); + instrumentation.addTransformer(new SourceSinkTransformer(), true); + } + + public static InstrumentationAdaptor getInstrumentation() { + return instrumentation; + } + + @InvokedViaInstrumentation(record = TaintMethodRecord.INSTRUMENT_CLASS_BYTES) + public static byte[] instrumentClassBytes(byte[] in) { + return new PCLoggingTransformer().transform(null, null, null, null, in, false); + } + + @InvokedViaInstrumentation(record = TaintMethodRecord.INSTRUMENT_CLASS_BYTES_ANONYMOUS) + public static byte[] instrumentClassBytesAnonymous(byte[] in) { + return new PCLoggingTransformer().transform(null, null, null, null, in, true); + } + + private static String[] parseOptions(String agentArgs) { + SinglyLinkedList options = new SinglyLinkedList<>(); + if (agentArgs != null && !agentArgs.isEmpty()) { + for (String arg : agentArgs.split(",")) { + int split = arg.indexOf('='); + if (split == -1) { + options.addLast("-" + arg); + } else { + options.addLast("-" + arg.substring(0, split)); + options.addLast(arg.substring(split + 1)); + } + } + } + if (Configuration.IS_JAVA_8) { + options.addLast("-java8"); + } + return options.toArray(new String[0]); + } +} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorBaseTransformer.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorBaseTransformer.java index 8a01d37ee..cc2e25313 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorBaseTransformer.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorBaseTransformer.java @@ -1,6 +1,5 @@ package edu.columbia.cs.psl.phosphor; -import java.lang.instrument.IllegalClassFormatException; import java.security.ProtectionDomain; /* Provides appropriate phosphor tagged versions of transform. */ @@ -9,21 +8,36 @@ public abstract class PhosphorBaseTransformer { public static boolean INITED = false; protected static int isBusyTransforming = 0; - public abstract byte[] transform(ClassLoader loader, final String className2, Class classBeingRedefined, - ProtectionDomain protectionDomain, byte[] classfileBuffer, boolean isAnonymousClassDefinition); + public abstract byte[] transform( + ClassLoader loader, + final String className2, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer, + boolean isAnonymousClassDefinition); - public byte[] signalAndTransform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, - byte[] classFileBuffer, boolean isAnonymousClassDefinition) throws IllegalClassFormatException { + public byte[] signalAndTransform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classFileBuffer, + boolean isAnonymousClassDefinition) { try { - synchronized(PhosphorBaseTransformer.class) { + synchronized (PhosphorBaseTransformer.class) { isBusyTransforming++; } - return transform(loader, className, classBeingRedefined, protectionDomain, classFileBuffer, isAnonymousClassDefinition); + return transform( + loader, + className, + classBeingRedefined, + protectionDomain, + classFileBuffer, + isAnonymousClassDefinition); } finally { - synchronized(PhosphorBaseTransformer.class) { + synchronized (PhosphorBaseTransformer.class) { isBusyTransforming--; } } } - } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PreMain.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PreMain.java deleted file mode 100644 index a6adb7441..000000000 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PreMain.java +++ /dev/null @@ -1,370 +0,0 @@ -package edu.columbia.cs.psl.phosphor; - -import edu.columbia.cs.psl.phosphor.instrumenter.*; -import edu.columbia.cs.psl.phosphor.instrumenter.asm.OffsetPreservingClassReader; -import edu.columbia.cs.psl.phosphor.org.objectweb.asm.commons.OurSerialVersionUIDAdder; -import edu.columbia.cs.psl.phosphor.runtime.StringUtils; -import edu.columbia.cs.psl.phosphor.runtime.TaintInstrumented; -import edu.columbia.cs.psl.phosphor.struct.SinglyLinkedList; -import edu.columbia.cs.psl.phosphor.struct.TaintedWithObjTag; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.HashSet; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.LinkedList; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.List; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.Set; -import org.objectweb.asm.*; -import org.objectweb.asm.tree.AnnotationNode; -import org.objectweb.asm.tree.ClassNode; -import org.objectweb.asm.tree.FieldNode; -import org.objectweb.asm.tree.MethodNode; -import org.objectweb.asm.util.CheckClassAdapter; -import org.objectweb.asm.util.TraceClassVisitor; - -import java.io.*; -import java.lang.reflect.Constructor; -import java.security.ProtectionDomain; - -public class PreMain { - - public static boolean DEBUG = System.getProperty("phosphor.debug") != null; - public static boolean RUNTIME_INST = false; - public static boolean INSTRUMENTATION_EXCEPTION_OCCURRED = false; - public static ClassLoader bigLoader = PreMain.class.getClassLoader(); - - private static InstrumentationHelper instrumentationHelper; - - private PreMain() { - // Prevents this class from being instantiated - } - - /** - * Used for Java 9+ - * @param args - * @param instrumentationHelper - */ - public static void premain(String args, InstrumentationHelper instrumentationHelper) { - PreMain.instrumentationHelper = instrumentationHelper; - instrumentationHelper.addTransformer(new ClassSupertypeReadingTransformer()); - RUNTIME_INST = true; - if(args != null) { - PhosphorOption.configure(true, parseArgs(args)); - } - if(System.getProperty("phosphorCacheDirectory") != null) { - Configuration.CACHE = TransformationCache.getInstance(System.getProperty("phosphorCacheDirectory")); - } - if(Instrumenter.loader == null) { - Instrumenter.loader = bigLoader; - } - // Ensure that BasicSourceSinkManager & anything needed to call isSourceOrSinkOrTaintThrough gets initialized - BasicSourceSinkManager.loadTaintMethods(); - BasicSourceSinkManager.getInstance().isSourceOrSinkOrTaintThrough(Object.class); - instrumentationHelper.addTransformer(new PCLoggingTransformer()); - instrumentationHelper.addTransformer(new SourceSinkTransformer(), true); - } - - private static String[] parseArgs(String argString) { - String[] args = argString.split(","); - SinglyLinkedList argList = new SinglyLinkedList<>(); - for(String arg : args) { - int split = arg.indexOf('='); - if(split == -1) { - argList.addLast("-" + arg); - } else { - String option = arg.substring(0, split); - String value = arg.substring(split + 1); - argList.addLast("-" + option); - argList.addLast(value); - } - } - return argList.toArray(new String[0]); - } - - public static InstrumentationHelper getInstrumentationHelper() { - return instrumentationHelper; - } - - @InvokedViaInstrumentation(record = TaintMethodRecord.INSTRUMENT_CLASS_BYTES) - public static byte[] instrumentClassBytes(byte[] in){ - return new PCLoggingTransformer().transform(null, null, null,null, in, false); - } - - @InvokedViaInstrumentation(record = TaintMethodRecord.INSTRUMENT_CLASS_BYTES_ANONYMOUS) - public static byte[] instrumentClassBytesAnonymous(byte[] in){ - return new PCLoggingTransformer().transform(null, null, null,null, in, true); - } - public static final class PCLoggingTransformer extends PhosphorBaseTransformer { - - public PCLoggingTransformer() { - TaintUtils.VERIFY_CLASS_GENERATION = System.getProperty("phosphor.verify") != null; - } - - @Override - public byte[] transform(ClassLoader loader, final String className2, Class classBeingRedefined, - ProtectionDomain protectionDomain, byte[] classfileBuffer, boolean isAnonymousClassDefinition) { - ClassReader cr = (Configuration.READ_AND_SAVE_BCI ? new OffsetPreservingClassReader(classfileBuffer) - : new ClassReader(classfileBuffer)); - String className = cr.getClassName(); - if (Instrumenter.isIgnoredClass(className)) { - switch (className) { - case "java/lang/Boolean": - case "java/lang/Byte": - case "java/lang/Character": - case "java/lang/Short": - return processBoolean(classfileBuffer); - } - return classfileBuffer; - } - Configuration.taintTagFactory.instrumentationStarting(className); - try { - ClassNode cn = new ClassNode(); - cr.accept(cn, ClassReader.SKIP_CODE); - boolean upgradeVersion = false; - if (className.equals("org/jruby/parser/Ruby20YyTables")) { - cn.version = 51; - upgradeVersion = true; - } - if (cn.invisibleAnnotations != null) { - for (Object o : cn.invisibleAnnotations) { - AnnotationNode an = (AnnotationNode) o; - if (an.desc.equals(Type.getDescriptor(TaintInstrumented.class))) { - return classfileBuffer; - } - } - } - if (cn.interfaces != null) { - for (Object s : cn.interfaces) { - if (s.equals(Type.getInternalName(TaintedWithObjTag.class))) { - return classfileBuffer; - } - } - } - for (Object mn : cn.methods) { - if (((MethodNode) mn).name.equals("getPHOSPHOR_TAG")) { - return classfileBuffer; - } - } - if (Configuration.CACHE != null) { - byte[] cachedClass = Configuration.CACHE.load(className, classfileBuffer); - if (cachedClass != null) { - return cachedClass; - } - } - if (DEBUG) { - try { - File debugDir = new File("debug-preinst"); - if (!debugDir.exists()) { - debugDir.mkdir(); - } - File f = new File("debug-preinst/" + className.replace("/", ".") + ".class"); - FileOutputStream fos = new FileOutputStream(f); - fos.write(classfileBuffer); - fos.close(); - } catch (IOException ex) { - ex.printStackTrace(); - } - } - boolean isiFace = (cn.access & Opcodes.ACC_INTERFACE) != 0; - List fields = new LinkedList<>(); - for (FieldNode node : cn.fields) { - fields.add(node); - } - boolean skipFrames = LegacyClassFixer.shouldFixFrames(cn, className, cr); - if (skipFrames) { - // This class is old enough to not guarantee frames. - // Generate new frames for analysis reasons, then make sure to not emit ANY frames. - cr = LegacyClassFixer.fix(cr); - } - try { - byte[] instrumentedBytes = instrumentWithRetry(cr, classfileBuffer, isiFace, className, skipFrames, - upgradeVersion, fields, null, false, isAnonymousClassDefinition); - if (DEBUG) { - File f = new File("debug/" + className + ".class"); - f.getParentFile().mkdirs(); - FileOutputStream fos = new FileOutputStream(f); - fos.write(instrumentedBytes); - fos.close(); - } - if (Configuration.CACHE != null) { - Configuration.CACHE.store(className, classfileBuffer, instrumentedBytes); - } - return instrumentedBytes; - } catch (Throwable ex) { - ex.printStackTrace(); - throw new IllegalStateException(ex); - } - } finally { - Configuration.taintTagFactory.instrumentationEnding(className); - } - } - - static byte[] instrumentWithRetry(ClassReader cr, byte[] classFileBuffer, boolean isiFace, String className, - boolean skipFrames, boolean upgradeVersion, List fields, - Set methodsToReduceSizeOf, boolean traceClass, boolean isAnonymousClassDefinition) { - TraceClassVisitor debugTracer = null; - try { - try { - ClassWriter cw = new HackyClassWriter(null, ClassWriter.COMPUTE_MAXS); - ClassVisitor _cv = cw; - if(traceClass) { - System.out.println("Saving " + className + " to debug-preinst/"); - File f = new File("debug-preinst/" + className.replace("/", ".") + - ".class"); - if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { - System.err.println("Failed to make debug directory: " + f); - } else { - try { - FileOutputStream fos = new FileOutputStream(f); - fos.write(classFileBuffer); - fos.close(); - } catch (Exception ex2) { - ex2.printStackTrace(); - } - } - debugTracer = new TraceClassVisitor(null, null); - _cv = debugTracer; - } - if(Configuration.POST_CLASS_VISITOR != null) { - try { - Constructor extra = Configuration.POST_CLASS_VISITOR - .getConstructor(ClassVisitor.class, Boolean.TYPE, byte[].class); - _cv = extra.newInstance(_cv, skipFrames, classFileBuffer); - } catch(Exception e) { - // - } - } - if(Configuration.extensionClassVisitor != null) { - Constructor extra = Configuration.extensionClassVisitor.getConstructor(ClassVisitor.class, Boolean.TYPE); - _cv = extra.newInstance(_cv, skipFrames); - } - if(DEBUG || TaintUtils.VERIFY_CLASS_GENERATION) { - _cv = new CheckClassAdapter(_cv, false); - } - if(SerializationFixingCV.isApplicable(className)) { - _cv = new SerializationFixingCV(_cv, className); - } - _cv = new ClinitRetransformClassVisitor(_cv); - boolean isGenerateConstructorAccessor = StringUtils.startsWith(className, "sun/reflect/GeneratedConstructor") - || StringUtils.startsWith(className, "jdk/internal/reflect/GeneratedConstructor"); // Calculating SVUID could trigger constructor generation causing stack overflow/infinite recursion - if(isiFace || isGenerateConstructorAccessor) { - _cv = new TaintTrackingClassVisitor(_cv, skipFrames, fields, methodsToReduceSizeOf, isAnonymousClassDefinition); - } else { - _cv = new OurSerialVersionUIDAdder(new TaintTrackingClassVisitor(_cv, skipFrames, fields, - methodsToReduceSizeOf, isAnonymousClassDefinition)); - } - if(EclipseCompilerCV.isEclipseCompilerClass(className)) { - _cv = new EclipseCompilerCV(_cv); - } - if(OgnlUtilCV.isOgnlUtilClass(className) && !Configuration.REENABLE_CACHES) { - _cv = new OgnlUtilCV(_cv); - } - if(JettyBufferUtilCV.isApplicable(className)) { - _cv = new JettyBufferUtilCV(_cv); - } - if(PowerMockUtilCV.isApplicable(className)) { - _cv = new PowerMockUtilCV(_cv); - } - if(Configuration.PRIOR_CLASS_VISITOR != null) { - try { - Constructor extra = Configuration.PRIOR_CLASS_VISITOR.getConstructor(ClassVisitor.class, Boolean.TYPE); - _cv = extra.newInstance(_cv, skipFrames); - } catch(Exception e) { - // - } - } - cr.accept(_cv, ClassReader.EXPAND_FRAMES); - byte[] instrumentedBytes = cw.toByteArray(); - if (!traceClass && (DEBUG || TaintUtils.VERIFY_CLASS_GENERATION)) { - ClassReader cr2 = new ClassReader(instrumentedBytes); - try { - cr2.accept(new CheckClassAdapter(new ClassWriter(0), true), ClassReader.EXPAND_FRAMES); - } catch (Throwable t) { - t.printStackTrace(); - File f = new File("debug-verify/" + className.replace("/", ".") + ".class"); - if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { - System.err.println("Failed to make debug directory: " + f); - } else { - try { - FileOutputStream fos = new FileOutputStream(f); - fos.write(instrumentedBytes); - fos.close(); - } catch (Exception ex2) { - ex2.printStackTrace(); - } - System.out.println("Saved broken class to " + f); - } - System.out.println("Saving uninstrumented " + className + " to debug-preinst/"); - f = new File("debug-preinst/" + className.replace("/", ".") + - ".class"); - if (!f.getParentFile().isDirectory() && !f.getParentFile().mkdirs()) { - System.err.println("Failed to make debug directory: " + f); - } else { - try { - FileOutputStream fos = new FileOutputStream(f); - fos.write(classFileBuffer); - fos.close(); - } catch (Exception ex2) { - ex2.printStackTrace(); - } - } - } - } - return instrumentedBytes; - } catch(MethodTooLargeException ex) { - if(methodsToReduceSizeOf == null) { - methodsToReduceSizeOf = new HashSet<>(); - } - if(!methodsToReduceSizeOf.add(ex.getMethodName() + ex.getDescriptor())){ - throw ex; //We already tried and failed to make this fit :( - } - return instrumentWithRetry(cr, classFileBuffer, isiFace, className, skipFrames, upgradeVersion, fields, methodsToReduceSizeOf, false, isAnonymousClassDefinition); - } - } catch (Throwable ex) { - INSTRUMENTATION_EXCEPTION_OCCURRED = true; - if (!traceClass) { - System.err.println("Exception occurred while instrumenting " + className + ":"); - ex.printStackTrace(); - instrumentWithRetry(cr, classFileBuffer, isiFace, className, skipFrames, upgradeVersion, fields, methodsToReduceSizeOf, true, isAnonymousClassDefinition); - return classFileBuffer; - } - ex.printStackTrace(); - System.err.println("method so far:"); - try { - PrintWriter pw = new PrintWriter(new FileWriter("lastClass.txt")); - debugTracer.p.print(pw); - pw.flush(); - } catch (IOException ex2) { - ex2.printStackTrace(); - } - return classFileBuffer; - } - } - - private static byte[] processBoolean(byte[] classFileBuffer) { - ClassReader cr = new ClassReader(classFileBuffer); - ClassNode cn = new ClassNode(Configuration.ASM_VERSION); - cr.accept(cn, 0); - boolean addField = true; - for(Object o : cn.fields) { - FieldNode fn = (FieldNode) o; - if(fn.name.equals("valueOf")) { - addField = false; - break; - } - } - for(Object o : cn.methods) { - MethodNode mn = (MethodNode) o; - if (mn.name.startsWith("toUpperCase") - || mn.name.startsWith("codePointAtImpl") - || mn.name.startsWith("codePointBeforeImpl")) { - mn.access = mn.access | Opcodes.ACC_PUBLIC; - } - } - if(addField) { - cn.fields.add(new FieldNode(Opcodes.ACC_PUBLIC, "valueOf", "Z", null, false)); - ClassWriter cw = new ClassWriter(0); - cn.accept(cw); - return cw.toByteArray(); - } - return classFileBuffer; - } - } -} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/SourceSinkTransformer.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/SourceSinkTransformer.java index 6f9c3ee5a..9d0306d2c 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/SourceSinkTransformer.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/SourceSinkTransformer.java @@ -3,7 +3,6 @@ import edu.columbia.cs.psl.phosphor.instrumenter.SourceSinkTaintingClassVisitor; import edu.columbia.cs.psl.phosphor.runtime.NonModifiableClassException; import edu.columbia.cs.psl.phosphor.struct.LinkedList; -import edu.columbia.cs.psl.phosphor.struct.harmony.util.HashMap; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; @@ -22,25 +21,6 @@ public class SourceSinkTransformer extends PhosphorBaseTransformer { // Whether classes are in the process of being retransformed or checked for retransformation private static boolean isBusyRetransforming = false; - public static void main(String[] args) { - PhosphorOption.configure(true, args); - - SourceSinkTransformer transformer = new SourceSinkTransformer(); - try { - ClassReader cr = new ClassReader(new FileInputStream("z.class")); - ClassWriter cw = new ClassWriter(0); - cr.accept(cw, 0); - byte[] ret = transformer.transform(null, "z.class", HashMap.class, null, cw.toByteArray(), false); - System.out.println(ret); - - - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - } - @Override public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer, boolean isAnonymousClassDefinition) { if(classBeingRedefined == null) { @@ -54,7 +34,7 @@ public byte[] transform(ClassLoader loader, String className, Class classBein ClassReader cr = new ClassReader(classfileBuffer); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); ClassVisitor cv = cw; - if (PreMain.DEBUG || TaintUtils.VERIFY_CLASS_GENERATION) { + if (Phosphor.DEBUG || TaintUtils.VERIFY_CLASS_GENERATION) { cv = new CheckClassAdapter(cw, false); } PrintWriter pw = new PrintWriter(new File("lastClass-sourceSink.txt")); @@ -66,7 +46,7 @@ public byte[] transform(ClassLoader loader, String className, Class classBein debug.p.print(pw); pw.close(); } - if (PreMain.DEBUG) { + if (Phosphor.DEBUG) { try { File f = new File("debug-source-sink/" + className + ".class"); f.getParentFile().mkdirs(); @@ -100,7 +80,7 @@ public static void retransform(Class clazz) { } // Check if PreMain's instrumentation has been set by a call to premain and that Configuration.init() has // been called to initialize the configuration - if(isBusyTransforming == 0 && !isBusyRetransforming && INITED && PreMain.getInstrumentationHelper() != null) { + if(isBusyTransforming == 0 && !isBusyRetransforming && INITED && Phosphor.getInstrumentation() != null) { isBusyRetransforming = true; retransformQueue.addFast(clazz); // Retransform clazz and any classes that were initialized before retransformation could occur. @@ -111,7 +91,7 @@ public static void retransform(Class clazz) { // poppedClazz represents a class or interface that is or is a subtype of a class or interface with // at least one method labeled as being a sink or source or taintThrough method if(!poppedClazz.equals(PrintStream.class)) { - PreMain.getInstrumentationHelper().retransformClasses(poppedClazz); + Phosphor.getInstrumentation().retransformClasses(poppedClazz); } } } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/agent/PhosphorAgent.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/agent/PhosphorAgent.java new file mode 100644 index 000000000..f10b85ae5 --- /dev/null +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/agent/PhosphorAgent.java @@ -0,0 +1,100 @@ +package edu.columbia.cs.psl.phosphor.agent; + +import edu.columbia.cs.psl.phosphor.Configuration; +import edu.columbia.cs.psl.phosphor.InstrumentationAdaptor; +import edu.columbia.cs.psl.phosphor.Phosphor; +import edu.columbia.cs.psl.phosphor.PhosphorBaseTransformer; +import edu.columbia.cs.psl.phosphor.runtime.NonModifiableClassException; +import edu.columbia.cs.psl.phosphor.runtime.PhosphorStackFrame; + +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.IllegalClassFormatException; +import java.lang.instrument.Instrumentation; +import java.lang.instrument.UnmodifiableClassException; +import java.security.ProtectionDomain; + +import static edu.columbia.cs.psl.phosphor.PhosphorBaseTransformer.INITED; + +public final class PhosphorAgent { + private PhosphorAgent() { + throw new AssertionError("Tried to instantiate static agent class: " + getClass()); + } + + @SuppressWarnings("unused") + public static void premain(String agentArgs, Instrumentation inst, PhosphorStackFrame frame) { + premain(agentArgs, inst); + } + + public static void premain(String agentArgs, Instrumentation inst) { + InstrumentationAdaptor adaptor = new InstrumentationAdaptor() { + private final Instrumentation delegate = inst; + + @Override + public void addTransformer(final PhosphorBaseTransformer transformer) { + delegate.addTransformer(new PhosphorTransformerBridge(transformer)); + } + + @Override + public Class[] getAllLoadedClasses() { + return delegate.getAllLoadedClasses(); + } + + @Override + public void addTransformer(PhosphorBaseTransformer transformer, boolean canRedefineClasses) { + delegate.addTransformer(new PhosphorTransformerBridge(transformer), canRedefineClasses); + } + + @Override + public void retransformClasses(Class clazz) throws NonModifiableClassException { + try { + delegate.retransformClasses(clazz); + } catch (UnmodifiableClassException e) { + throw new NonModifiableClassException(e); + } + } + }; + Phosphor.initialize(agentArgs, adaptor); + } + + private static final class PhosphorTransformerBridge implements ClassFileTransformer { + private final PhosphorBaseTransformer transformer; + + public PhosphorTransformerBridge(PhosphorBaseTransformer transformer) { + this.transformer = transformer; + } + + @SuppressWarnings("unused") + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer, + PhosphorStackFrame frame) + throws IllegalClassFormatException { + return transform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer); + } + + @Override + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer) + throws IllegalClassFormatException { + try { + if (!INITED) { + Configuration.init(); + INITED = true; + } + return transformer.signalAndTransform( + loader, className, classBeingRedefined, protectionDomain, classfileBuffer, false); + } catch (Throwable t) { + //noinspection CallToPrintStackTrace + t.printStackTrace(); + throw t; + } + } + } +} diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/ConfigurationEmbeddingMV.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/ConfigurationEmbeddingMV.java index d661333ac..3d9ed7471 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/ConfigurationEmbeddingMV.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/ConfigurationEmbeddingMV.java @@ -11,7 +11,6 @@ import static org.objectweb.asm.Opcodes.ACC_PUBLIC; import static org.objectweb.asm.Opcodes.ACONST_NULL; -import static org.objectweb.asm.Opcodes.ASM9; import static org.objectweb.asm.Opcodes.DUP; import static org.objectweb.asm.Opcodes.INVOKEINTERFACE; import static org.objectweb.asm.Opcodes.INVOKESPECIAL; @@ -20,7 +19,7 @@ public class ConfigurationEmbeddingMV extends MethodVisitor { public ConfigurationEmbeddingMV(MethodVisitor mv) { - super(ASM9, mv); + super(Configuration.ASM_VERSION, mv); } // Embed initialized Configuration class into class file. diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/TaintMethodRecord.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/TaintMethodRecord.java index 38e6e8c4c..f0259f787 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/TaintMethodRecord.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/instrumenter/TaintMethodRecord.java @@ -1,6 +1,6 @@ package edu.columbia.cs.psl.phosphor.instrumenter; -import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.Phosphor; import edu.columbia.cs.psl.phosphor.TaintUtils; import edu.columbia.cs.psl.phosphor.control.ControlFlowStack; import edu.columbia.cs.psl.phosphor.runtime.*; @@ -53,10 +53,8 @@ public enum TaintMethodRecord implements MethodRecord { PREPARE_FOR_CALL_REFLECTIVE_CONSTRUCTOR(INVOKESTATIC, ReflectionMasker.class, "prepareForCall", ReflectionMasker.ConstructorInvocationPair.class, false, Constructor.class, Object[].class, PhosphorStackFrame.class), IS_INSTANCE(INVOKESTATIC, ReflectionMasker.class, "isInstance", Boolean.TYPE, false, Class.class, Object.class, PhosphorStackFrame.class), - INSTRUMENT_CLASS_BYTES(INVOKESTATIC, PreMain.class, "instrumentClassBytes", byte[].class, false, byte[].class), - INSTRUMENT_CLASS_BYTES_ANONYMOUS(INVOKESTATIC, PreMain.class, "instrumentClassBytesAnonymous", byte[].class, false, byte[].class), - - + INSTRUMENT_CLASS_BYTES(INVOKESTATIC, Phosphor.class, "instrumentClassBytes", byte[].class, false, byte[].class), + INSTRUMENT_CLASS_BYTES_ANONYMOUS(INVOKESTATIC, Phosphor.class, "instrumentClassBytesAnonymous", byte[].class, false, byte[].class), //Phosphor Stack Frame START_STACK_FRAME_TRACKING(INVOKESTATIC, PhosphorStackFrame.class, "initialize", Void.TYPE, false), @@ -99,9 +97,6 @@ public enum TaintMethodRecord implements MethodRecord { GET_RETURN_WRAPPER_SHORT(INVOKEVIRTUAL, PhosphorStackFrame.class, "getReturnWrapper", TaggedShortArray.class, false, short[].class), GET_RETURN_WRAPPER_DOUBLE(INVOKEVIRTUAL, PhosphorStackFrame.class, "getReturnWrapper", TaggedDoubleArray.class, false, double[].class), - - - // Tainted array set methods TAINTED_BOOLEAN_ARRAY_SET(INVOKEVIRTUAL, TaggedBooleanArray.class, "set", Void.TYPE, false, int.class, boolean.class, Taint.class, Taint.class, PhosphorStackFrame.class), TAINTED_BYTE_ARRAY_SET(INVOKEVIRTUAL, TaggedByteArray.class, "set", Void.TYPE, false, int.class, byte.class, Taint.class, Taint.class, PhosphorStackFrame.class), diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/StringUtils.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/StringUtils.java index 78e037204..7a94a9bea 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/StringUtils.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/StringUtils.java @@ -2,7 +2,7 @@ import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.Phosphor; import edu.columbia.cs.psl.phosphor.runtime.proxied.InstrumentedJREFieldHelper; /** @@ -96,7 +96,7 @@ public static boolean _startsWithJava8(String thisStr, String string) { } public static boolean startsWith(String str, String prefix) { - if (PreMain.RUNTIME_INST) { + if (Phosphor.RUNTIME_INST) { if(Configuration.IS_JAVA_8){ return _startsWithJava8(str, prefix); } else { @@ -112,7 +112,7 @@ public static boolean equals(String s1, String s2) { } else if (s1 == null || s2 == null) { return false; } else { - if (PreMain.RUNTIME_INST) { + if (Phosphor.RUNTIME_INST) { if (Configuration.IS_JAVA_8) { return _equalsJava8(s1, s2); } else { diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported/RuntimeSunMiscUnsafePropagator.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported/RuntimeSunMiscUnsafePropagator.java index 7bb92c4ea..7c6d10864 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported/RuntimeSunMiscUnsafePropagator.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported/RuntimeSunMiscUnsafePropagator.java @@ -1,7 +1,7 @@ package edu.columbia.cs.psl.phosphor.runtime.jdk.unsupported; import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.Phosphor; import edu.columbia.cs.psl.phosphor.TaintUtils; import edu.columbia.cs.psl.phosphor.runtime.MultiDArrayUtils; import edu.columbia.cs.psl.phosphor.runtime.PhosphorStackFrame; @@ -793,7 +793,7 @@ public static long getLongVolatile(UnsafeProxy unsafe, Object obj, long offset, } public static Class defineClass(UnsafeProxy unsafe, String var1, byte[] var2, int var3, int var4, ClassLoader var5, ProtectionDomain var6, PhosphorStackFrame phosphorStackFrame) { - byte[] instrumented = PreMain.instrumentClassBytes(var2); + byte[] instrumented = Phosphor.instrumentClassBytes(var2); return unsafe.defineClass(var1, instrumented, 0, instrumented.length, var5, var6); } private enum SpecialAccessPolicy { diff --git a/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java b/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java index 97efc245f..efdc202d3 100644 --- a/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java +++ b/Phosphor/src/test/java/edu/columbia/cs/psl/phosphor/InstrumenterTest.java @@ -14,12 +14,12 @@ public class InstrumenterTest { @Before public void clearErrorFlag() { - PreMain.INSTRUMENTATION_EXCEPTION_OCCURRED = false; + Phosphor.INSTRUMENTATION_EXCEPTION_OCCURRED = false; } @After public void checkForError() { - if (PreMain.INSTRUMENTATION_EXCEPTION_OCCURRED) { + if (Phosphor.INSTRUMENTATION_EXCEPTION_OCCURRED) { Assert.fail("Instrumentation error occurred"); } } @@ -43,7 +43,7 @@ public void testAggressivelyReduceMethodSizeGetStaticThenArrayLength() { mv.visitEnd(); cw.visitEnd(); byte[] classFileBuffer = cw.toByteArray(); - new PreMain.PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); + new PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); } @Test @@ -87,6 +87,6 @@ public void wideDuplicates() { mv.visitMaxs(-1, -1); mv.visitEnd(); byte[] classFileBuffer = cw.toByteArray(); - new PreMain.PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); + new PCLoggingTransformer().transform(null, null, null, null, classFileBuffer, false); } } \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 10beb587f..f6eaacb4e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -18,7 +18,6 @@ ${project.build.directory}/phosphor/data/cache false ${edu.gmu.swe.phosphor:Phosphor:jar} - ${edu.gmu.swe.phosphor:phosphor-jigsaw-javaagent:jar} false @@ -115,7 +114,6 @@ true - ${edu.gmu.swe.phosphor:Phosphor:jar} @@ -147,7 +145,7 @@ ${project.build.directory} ${phosphor.jar} - ${phosphor.agent} + ${phosphor.jar} ${data.flow.java}/bin/java ${isJava8} @@ -196,7 +194,7 @@ -DphosphorCacheDirectory=${data.flow.cache} -Xbootclasspath/a:${phosphor.jar} - -javaagent:${phosphor.agent}=${auto.taint},enum,acmpeq + -javaagent:${phosphor.jar}=${auto.taint},enum,acmpeq @@ -242,7 +240,7 @@ false -Xbootclasspath/a:${phosphor.jar} - -javaagent:${phosphor.agent}=${auto.taint},enum,acmpeq + -javaagent:${phosphor.jar}=${auto.taint},enum,acmpeq diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index a82206ff8..d39923466 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -7,12 +7,13 @@ INST_JVM=$4 IS_JAVA_8=$5 DACAPO_DIR=$BUILD_DIR/dacapo INST_DACAPO_DIR=$BUILD_DIR/dacapo-inst -BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) #tradebeans tradesoap are disabled in this script because of the PITA with distributing the openjdk jce.jar file with everything, then switching it in... +BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) +#tradebeans tradesoap are disabled in this script because of the difficulty with distributing the openjdk jce.jar file with everything, then switching it in... #we don't run batik anymore because it's an enormous pain to automatically set up oracle java... #tomcat started crashing on travis #if you want to run them, copy the jce.jar file from an openjdk distribution of the same version into your hotspot jvm that you are going to instrument before instrumenting it. # (export controls are annoying) -# eclipse doesn't work wihtout big hacks on travis +# eclipse doesn't work without big hacks on travis # ..also disable lusearch because travis seems unhappy when we run it there HAD_ERROR=0 if [ ! -d "$DACAPO_DIR" ]; then diff --git a/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/BasePhosphorTest.java b/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/BasePhosphorTest.java index 5559ade66..d802dff26 100644 --- a/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/BasePhosphorTest.java +++ b/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/BasePhosphorTest.java @@ -1,6 +1,6 @@ package edu.columbia.cs.psl.test.phosphor; -import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.Phosphor; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -8,12 +8,12 @@ public class BasePhosphorTest { @Before public void clearErrorFlag() { - PreMain.INSTRUMENTATION_EXCEPTION_OCCURRED = false; + Phosphor.INSTRUMENTATION_EXCEPTION_OCCURRED = false; } @After public void checkForError() { - if(PreMain.INSTRUMENTATION_EXCEPTION_OCCURRED) { + if(Phosphor.INSTRUMENTATION_EXCEPTION_OCCURRED) { Assert.fail("Instrumentation error occurred"); } } diff --git a/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/runtime/ReflectionImplicitITCase.java b/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/runtime/ReflectionImplicitITCase.java index a5c276835..50850f869 100644 --- a/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/runtime/ReflectionImplicitITCase.java +++ b/integration-tests/src/test/java/edu/columbia/cs/psl/test/phosphor/runtime/ReflectionImplicitITCase.java @@ -10,10 +10,9 @@ import java.util.Set; import java.util.stream.IntStream; +import edu.columbia.cs.psl.phosphor.Phosphor; import org.junit.Test; -import edu.columbia.cs.psl.phosphor.PreMain; - public class ReflectionImplicitITCase extends ReflectionObjTagITCase{ @Test @@ -33,7 +32,7 @@ public void testMultiDArrayAssignableFrom() throws Exception { @Test public void testReflectionDoesntCrash() { try { - for (Class c : PreMain.getInstrumentationHelper().getAllLoadedClasses()) { + for (Class c : Phosphor.getInstrumentation().getAllLoadedClasses()) { Set allFields = new HashSet(); try { Field[] declaredFields = c.getDeclaredFields(); diff --git a/phosphor-distribution/pom.xml b/phosphor-distribution/pom.xml deleted file mode 100644 index eb068e790..000000000 --- a/phosphor-distribution/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - 4.0.0 - phosphor-distribution - pom - Distribution of Phosphor main jar for Java 8, plus Jigsaw (Java9+) JVM instrumenter and runtime agent - - edu.gmu.swe.phosphor - phosphor-parent - 0.1.0-SNAPSHOT - - - - - maven-assembly-plugin - 3.2.0 - - - distro-assembly - package - - single - - - - src/assembly/bin.xml - - - - - - - - - - edu.gmu.swe.phosphor - Phosphor - ${project.version} - - - edu.gmu.swe.phosphor - phosphor-instrument-jigsaw - ${project.version} - - - edu.gmu.swe.phosphor - phosphor-jigsaw-javaagent - ${project.version} - - - diff --git a/phosphor-distribution/src/assembly/bin.xml b/phosphor-distribution/src/assembly/bin.xml deleted file mode 100644 index 24d05b8f3..000000000 --- a/phosphor-distribution/src/assembly/bin.xml +++ /dev/null @@ -1,23 +0,0 @@ - - bin - - dir - - false - - - true - - edu.gmu.swe.phosphor:Phosphor - edu.gmu.swe.phosphor:phosphor-instrument-jigsaw - edu.gmu.swe.phosphor:phosphor-jigsaw-javaagent - - - false - false - - - - diff --git a/phosphor-instrument-jigsaw/pom.xml b/phosphor-driver/pom.xml similarity index 90% rename from phosphor-instrument-jigsaw/pom.xml rename to phosphor-driver/pom.xml index e7720654c..75706eb22 100644 --- a/phosphor-instrument-jigsaw/pom.xml +++ b/phosphor-driver/pom.xml @@ -1,8 +1,10 @@ 4.0.0 - phosphor-instrument-jigsaw - Tools for assisting in the creation of instrumented JVMs + phosphor-driver + + A driver for applying Phosphor's bytecode instrumentation to Java classes. + edu.gmu.swe.phosphor phosphor-parent @@ -59,10 +61,10 @@ - edu.gmu.swe.phosphor.instrument.JLinkRegistrationAgent + edu.columbia.cs.psl.phosphor.driver.JLinkRegistrationAgent - edu.gmu.swe.phosphor.instrument.PhosphorInstrumenter + edu.columbia.cs.psl.phosphor.driver.PhosphorInstrumenter @@ -83,18 +85,19 @@ org.jacoco.core - edu.gmu.swe.phosphor.instrument.org.jacoco.core + edu.columbia.cs.psl.phosphor.driver.org.jacoco.core org.objectweb.asm - edu.gmu.swe.phosphor.instrument.org.objectweb.asm + edu.columbia.cs.psl.phosphor.driver.org.objectweb.asm + edu.gmu.swe.phosphor:* org.jacoco:* org.ow2.asm:* diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/DeletingFileVisitor.java similarity index 96% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/DeletingFileVisitor.java index 3fef97a16..96f2109cc 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/DeletingFileVisitor.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/DeletingFileVisitor.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentJLinkPlugin.java similarity index 98% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentJLinkPlugin.java index 53debf655..e4d14a193 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentJLinkPlugin.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentJLinkPlugin.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import jdk.tools.jlink.plugin.Plugin; import jdk.tools.jlink.plugin.ResourcePool; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentUtil.java similarity index 98% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentUtil.java index 0c6fb2b1e..1312ad2e5 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentUtil.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/InstrumentUtil.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.io.*; import java.security.MessageDigest; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumentation.java similarity index 97% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumentation.java index 870ab558b..427105905 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumentation.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumentation.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.io.File; import java.io.IOException; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java similarity index 99% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java index cb7158a3e..83170c129 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Instrumenter.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import org.jacoco.core.internal.InputStreams; import org.jacoco.core.internal.instr.SignatureRemover; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkInvoker.java similarity index 98% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkInvoker.java index 7f1017ba6..45437af59 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkInvoker.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkInvoker.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.io.File; import java.io.FileWriter; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkRegistrationAgent.java similarity index 99% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkRegistrationAgent.java index 3512d4ce7..356407268 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/JLinkRegistrationAgent.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/JLinkRegistrationAgent.java @@ -204,7 +204,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.lang.instrument.Instrumentation; import java.util.*; @@ -215,7 +215,7 @@ */ @SuppressWarnings("Since15") public final class JLinkRegistrationAgent { - public static final String MODULE_NAME = "edu.gmu.swe.phosphor.instrument"; + public static final String MODULE_NAME = "edu.columbia.cs.psl.phosphor.driver"; private static final String JLINK_MODULE_NAME = "jdk.jlink"; private static final String JLINK_PLUGIN_PACKAGE_NAME = "jdk.tools.jlink.plugin"; private static final String JLINK_PLUGIN_CLASS_NAME = JLINK_PLUGIN_PACKAGE_NAME + ".Plugin"; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Packer.java similarity index 99% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Packer.java index ec124b62c..82c0cdcaa 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Packer.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Packer.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import jdk.tools.jlink.plugin.ResourcePool; import jdk.tools.jlink.plugin.ResourcePoolBuilder; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Patcher.java similarity index 75% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Patcher.java index 406049b16..5c3af3ac2 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/Patcher.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Patcher.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import java.io.IOException; diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumentation.java similarity index 84% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumentation.java index 6494fe1e2..eb4930b05 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumentation.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumentation.java @@ -1,9 +1,7 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; -import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.PhosphorOption; -import edu.columbia.cs.psl.phosphor.PhosphorPatcher; -import edu.columbia.cs.psl.phosphor.PreMain; +import edu.columbia.cs.psl.phosphor.*; +import edu.columbia.cs.psl.phosphor.agent.PhosphorAgent; import edu.columbia.cs.psl.phosphor.instrumenter.TaintTrackingClassVisitor; import edu.columbia.cs.psl.phosphor.org.apache.commons.cli.CommandLine; @@ -24,7 +22,7 @@ */ @SuppressWarnings("unused") public class PhosphorInstrumentation implements Instrumentation { - private PreMain.PCLoggingTransformer transformer; + private PCLoggingTransformer transformer; private Set classPathElements; @Override @@ -44,9 +42,9 @@ public void initialize(CommandLine line) throws IOException { setUpClassLoader(source); Configuration.init(); TaintTrackingClassVisitor.IS_RUNTIME_INST = false; - transformer = new PreMain.PCLoggingTransformer(); + transformer = new PCLoggingTransformer(); classPathElements = new HashSet<>(); - classPathElements.add(InstrumentUtil.getClassPathElement(PreMain.class)); + classPathElements.add(InstrumentUtil.getClassPathElement(PhosphorAgent.class)); configurationClasses.stream() .map(InstrumentUtil::getClassPathElement) .forEach(classPathElements::add); @@ -71,9 +69,10 @@ public Patcher createPatcher(Function entryLocator) { @Override public boolean shouldPack(String classFileName) { - return !classFileName.startsWith("edu/columbia/cs/psl/jigsaw/phosphor/instrumenter") - && !classFileName.endsWith("module-info.class") + return !classFileName.endsWith("module-info.class") && !classFileName.startsWith("org/") + && !classFileName.startsWith("edu/columbia/cs/psl/phosphor/agent/") + && !classFileName.startsWith("edu/columbia/cs/psl/phosphor/driver/") && !classFileName.startsWith("edu/columbia/cs/psl/phosphor/runtime/jdk/unsupported") && classFileName.endsWith(".class"); } @@ -99,6 +98,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO urls.add(input.toURI().toURL()); } urls.add(input.toURI().toURL()); - PreMain.bigLoader = new URLClassLoader(urls.toArray(new URL[0]), Instrumenter.class.getClassLoader()); + Phosphor.bigLoader = new URLClassLoader(urls.toArray(new URL[0]), Instrumenter.class.getClassLoader()); } } diff --git a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java similarity index 96% rename from phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java rename to phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java index d59e92dc0..f445b1401 100644 --- a/phosphor-instrument-jigsaw/src/main/java/edu/gmu/swe/phosphor/instrument/PhosphorInstrumenter.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java @@ -1,4 +1,4 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.driver; import edu.columbia.cs.psl.phosphor.Configuration; import edu.columbia.cs.psl.phosphor.PhosphorOption; diff --git a/phosphor-driver/src/main/java/module-info.java b/phosphor-driver/src/main/java/module-info.java new file mode 100644 index 000000000..740b12513 --- /dev/null +++ b/phosphor-driver/src/main/java/module-info.java @@ -0,0 +1,6 @@ +module edu.columbia.cs.psl.phosphor.driver { + exports edu.columbia.cs.psl.phosphor.driver; + + requires jdk.jlink; + requires java.instrument; +} diff --git a/phosphor-instrument-jigsaw/src/main/java/module-info.java b/phosphor-instrument-jigsaw/src/main/java/module-info.java deleted file mode 100644 index e154ca28a..000000000 --- a/phosphor-instrument-jigsaw/src/main/java/module-info.java +++ /dev/null @@ -1,5 +0,0 @@ -module edu.gmu.swe.phosphor.instrument { - exports edu.gmu.swe.phosphor.instrument; - requires jdk.jlink; - requires java.instrument; -} diff --git a/phosphor-instrument-maven-plugin/pom.xml b/phosphor-instrument-maven-plugin/pom.xml index 9cf17a6c9..ed455ea15 100644 --- a/phosphor-instrument-maven-plugin/pom.xml +++ b/phosphor-instrument-maven-plugin/pom.xml @@ -28,7 +28,7 @@ edu.gmu.swe.phosphor - phosphor-instrument-jigsaw + phosphor-driver ${project.version} diff --git a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java b/phosphor-instrument-maven-plugin/src/main/java/edu/columbia/cs/psl/phosphor/plugin/InstrumentMojo.java similarity index 96% rename from phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java rename to phosphor-instrument-maven-plugin/src/main/java/edu/columbia/cs/psl/phosphor/plugin/InstrumentMojo.java index 77d7fcd1c..702f1f7d5 100644 --- a/phosphor-instrument-maven-plugin/src/main/java/edu/gmu/swe/phosphor/instrument/InstrumentMojo.java +++ b/phosphor-instrument-maven-plugin/src/main/java/edu/columbia/cs/psl/phosphor/plugin/InstrumentMojo.java @@ -1,5 +1,9 @@ -package edu.gmu.swe.phosphor.instrument; +package edu.columbia.cs.psl.phosphor.plugin; +import edu.columbia.cs.psl.phosphor.driver.DeletingFileVisitor; +import edu.columbia.cs.psl.phosphor.driver.InstrumentUtil; +import edu.columbia.cs.psl.phosphor.driver.Instrumentation; +import edu.columbia.cs.psl.phosphor.driver.Instrumenter; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -36,7 +40,7 @@ * configured using the specified {@link InstrumentMojo#options}. * This instance determines the type of instrumentation applied. * - * @see edu.gmu.swe.phosphor.instrument.Instrumentation + * @see Instrumentation */ @Mojo(name = "instrument", defaultPhase = LifecyclePhase.PROCESS_TEST_RESOURCES) public class InstrumentMojo extends AbstractMojo { @@ -76,7 +80,7 @@ public class InstrumentMojo extends AbstractMojo { */ @Parameter( property = "phosphor.instrumentationType", - defaultValue = "edu.gmu.swe.phosphor.instrument.PhosphorInstrumentation") + defaultValue = "edu.columbia.cs.psl.phosphor.driver.PhosphorInstrumentation") private String instrumentationType; /** * Options passed to {@link Instrumentation#configure}. diff --git a/phosphor-jigsaw-javaagent/pom.xml b/phosphor-jigsaw-javaagent/pom.xml deleted file mode 100644 index 4c4d9849d..000000000 --- a/phosphor-jigsaw-javaagent/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - 4.0.0 - phosphor-jigsaw-javaagent - jar - A runtime instrumenter to go with Phosphor-instrumented JVMs. This version is only used for Java 9+ VMs - - edu.gmu.swe.phosphor - phosphor-parent - 0.1.0-SNAPSHOT - - - - - maven-compiler-plugin - 3.8.0 - - 9 - 9 - - - --add-reads=edu.columbia.cs.psl.jigsaw.phosphor.javaagent=ALL-UNNAMED - - - true - - - - org.apache.maven.plugins - maven-jar-plugin - - - - edu.columbia.cs.psl.jigsaw.phosphor.javaagent.PreMain - true - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.1 - - true - - - - attach-javadoc - - jar - - - - - - - - - release-sign-artifacts - - - gpg.passphrase - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.4 - - - sign-artifacts - verify - - sign - - - - - - - - - - - edu.gmu.swe.phosphor - Phosphor - ${project.version} - - - diff --git a/phosphor-jigsaw-javaagent/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/javaagent/PreMain.java b/phosphor-jigsaw-javaagent/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/javaagent/PreMain.java deleted file mode 100644 index 090c138ad..000000000 --- a/phosphor-jigsaw-javaagent/src/main/java/edu/columbia/cs/psl/jigsaw/phosphor/javaagent/PreMain.java +++ /dev/null @@ -1,75 +0,0 @@ -package edu.columbia.cs.psl.jigsaw.phosphor.javaagent; - -import edu.columbia.cs.psl.phosphor.Configuration; -import edu.columbia.cs.psl.phosphor.InstrumentationHelper; -import edu.columbia.cs.psl.phosphor.PhosphorBaseTransformer; -import edu.columbia.cs.psl.phosphor.runtime.NonModifiableClassException; -import edu.columbia.cs.psl.phosphor.runtime.PhosphorStackFrame; - -import java.lang.instrument.ClassFileTransformer; -import java.lang.instrument.IllegalClassFormatException; -import java.lang.instrument.Instrumentation; -import java.lang.instrument.UnmodifiableClassException; -import java.security.ProtectionDomain; - -import static edu.columbia.cs.psl.phosphor.PhosphorBaseTransformer.INITED; - -public class PreMain { - static class PhosphorTransformerBridge implements ClassFileTransformer { - private PhosphorBaseTransformer transformer; - - public PhosphorTransformerBridge(PhosphorBaseTransformer transformer) { - this.transformer = transformer; - } - - public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer, PhosphorStackFrame stackFrame) throws IllegalClassFormatException { - return transform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer); - } - - @Override - public byte[] transform(ClassLoader loader, String className, Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException { - try { - if (!INITED) { - Configuration.init(); - INITED = true; - } - return transformer.signalAndTransform(loader, className, classBeingRedefined, protectionDomain, classfileBuffer, false); - } catch (Throwable t) { - t.printStackTrace(); - throw t; - } - } - } - - public static void premain(String args, final Instrumentation instr, final PhosphorStackFrame stackFrame) { - premain(args, instr); - } - - public static void premain(String args, final Instrumentation instr) { - edu.columbia.cs.psl.phosphor.PreMain.premain(args, new InstrumentationHelper() { - @Override - public void addTransformer(final PhosphorBaseTransformer transformer) { - instr.addTransformer(new PhosphorTransformerBridge(transformer)); - } - - @Override - public Class[] getAllLoadedClasses() { - return instr.getAllLoadedClasses(); - } - - @Override - public void addTransformer(PhosphorBaseTransformer transformer, boolean canRedefineClasses) { - instr.addTransformer(new PhosphorTransformerBridge(transformer), canRedefineClasses); - } - - @Override - public void retransformClasses(Class clazz) throws NonModifiableClassException { - try { - instr.retransformClasses(clazz); - } catch (UnmodifiableClassException e) { - throw new NonModifiableClassException(e); - } - } - }); - } -} diff --git a/phosphor-jigsaw-javaagent/src/main/java/module-info.java b/phosphor-jigsaw-javaagent/src/main/java/module-info.java deleted file mode 100644 index 54c2b3f41..000000000 --- a/phosphor-jigsaw-javaagent/src/main/java/module-info.java +++ /dev/null @@ -1,4 +0,0 @@ -module edu.columbia.cs.psl.jigsaw.phosphor.javaagent { - exports edu.columbia.cs.psl.jigsaw.phosphor.javaagent; - requires java.instrument; -} diff --git a/pom.xml b/pom.xml index 15275c612..d8267aab4 100644 --- a/pom.xml +++ b/pom.xml @@ -25,12 +25,10 @@ Phosphor - phosphor-jigsaw-javaagent - phosphor-instrument-jigsaw + phosphor-driver phosphor-instrument-maven-plugin integration-tests phosphor-microbench - phosphor-distribution 1.8 From 6919edd49aabaf28ede0da10179dc2f27d51475f Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 16:05:14 -0500 Subject: [PATCH 08/12] * Finished driver --- integration-tests/pom.xml | 19 +++----- integration-tests/runDacapo.sh | 46 +++++++++---------- .../phosphor/driver/PhosphorInstrumenter.java | 1 + 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index f6eaacb4e..9d68e5ba8 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -18,7 +18,7 @@ ${project.build.directory}/phosphor/data/cache false ${edu.gmu.swe.phosphor:Phosphor:jar} - false + ${edu.gmu.swe.phosphor:phosphor-driver:jar} @@ -26,6 +26,11 @@ Phosphor ${project.version} + + edu.gmu.swe.phosphor + phosphor-driver + ${project.version} + edu.gmu.swe.phosphor phosphor-jigsaw-javaagent @@ -107,15 +112,6 @@ - - java8 - - 1.8 - - - true - - dacapo @@ -145,9 +141,8 @@ ${project.build.directory} ${phosphor.jar} - ${phosphor.jar} + ${phosphor.driver.jar} ${data.flow.java}/bin/java - ${isJava8} diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index d39923466..e267acce1 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -1,13 +1,12 @@ #!/bin/bash set -x -BUILD_DIR=$1 -PHOSPHOR_JAR=$2 -PHOSPHOR_JAVA_AGENT=$3 -INST_JVM=$4 -IS_JAVA_8=$5 -DACAPO_DIR=$BUILD_DIR/dacapo -INST_DACAPO_DIR=$BUILD_DIR/dacapo-inst -BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) +readonly BUILD_DIR=$1 +readonly PHOSPHOR_JAR=$2 +readonly DRIVER_JAR=$3 +readonly INST_JVM=$4 +readonly DACAPO_DIR=$BUILD_DIR/dacapo +readonly INST_DACAPO_DIR=$BUILD_DIR/dacapo-inst +readonly BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) #tradebeans tradesoap are disabled in this script because of the difficulty with distributing the openjdk jce.jar file with everything, then switching it in... #we don't run batik anymore because it's an enormous pain to automatically set up oracle java... #tomcat started crashing on travis @@ -15,39 +14,38 @@ BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) # (export controls are annoying) # eclipse doesn't work without big hacks on travis # ..also disable lusearch because travis seems unhappy when we run it there -HAD_ERROR=0 if [ ! -d "$DACAPO_DIR" ]; then echo "Downloading dacapo jar" - mkdir -p $DACAPO_DIR - cd $DACAPO_DIR + mkdir -p "$DACAPO_DIR" + cd "$DACAPO_DIR" wget --quiet https://www.jonbell.net/dacapo-9.12-MR1-bach.jar unzip -qq dacapo-9.12-MR1-bach.jar fi -cd $BUILD_DIR +cd "$BUILD_DIR" if [ ! -d "$INST_DACAPO_DIR" ]; then echo "Creating data flow instrumented dacapo" - java -Xmx6g -XX:MaxPermSize=512m -jar "$PHOSPHOR_JAR" -q -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" + java -Xmx6g -jar "$DRIVER_JAR" -q -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" else echo "Not regenerating data flow instrumented dacapo" fi - +errors=0 echo "Trying data flow benchmarks, one iteration only" for bm in "${BENCHMARKS[@]}"; do -# echo "$JAVA_HOME/bin/java -cp $DACAPO_DIR Harness $bm" - - if [ "$IS_JAVA_8" == "true" ]; then - $INST_JVM -Xmx1g -Xbootclasspath/a:"$PHOSPHOR_JAR" -javaagent:"$PHOSPHOR_JAVA_AGENT" -cp "$INST_DACAPO_DIR" -Declipse.java.home="$JAVA_HOME" -Djava.awt.headless=true Harness "$bm" - else - $INST_JVM -Xmx4g -javaagent:"$PHOSPHOR_JAVA_AGENT" -cp "$INST_DACAPO_DIR" -Declipse.java.home="$JAVA_HOME" -Djava.awt.headless=true Harness "$bm" - fi + $INST_JVM -Xmx4g \ + -Xbootclasspath/a:"$PHOSPHOR_JAR" \ + -javaagent:"$PHOSPHOR_JAR" \ + -cp "$INST_DACAPO_DIR" \ + -Declipse.java.home="$JAVA_HOME" \ + -Djava.awt.headless=true \ + Harness "$bm" if [ $? -ne 0 ]; then - HAD_ERROR=$(expr $HAD_ERROR + 1) + errors=var=$((errors + 1)) echo "ERROR!!!" fi done -echo "Total errors: $HAD_ERROR" -if [ $HAD_ERROR -ne 0 ]; then +echo "Total errors: $errors" +if [ $errors -ne 0 ]; then exit 255 fi diff --git a/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java index f445b1401..d07713f17 100644 --- a/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/PhosphorInstrumenter.java @@ -19,6 +19,7 @@ public static void main(String[] args) throws IOException { // The "help" option was specified return; } + args = line.getArgs(); File source = new File(args[0]); File destination = new File(args[1]); instrumentation.initialize(line); From 4012146d7220c3cc0a2157947b314b3d750fb264 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 16:12:19 -0500 Subject: [PATCH 09/12] * Fixed dependency issue --- integration-tests/pom.xml | 5 ----- integration-tests/runDacapo.sh | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 9d68e5ba8..fef0f7c5e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -31,11 +31,6 @@ phosphor-driver ${project.version} - - edu.gmu.swe.phosphor - phosphor-jigsaw-javaagent - ${project.version} - diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index e267acce1..136291e73 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -25,7 +25,7 @@ cd "$BUILD_DIR" if [ ! -d "$INST_DACAPO_DIR" ]; then echo "Creating data flow instrumented dacapo" - java -Xmx6g -jar "$DRIVER_JAR" -q -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" + java -Xmx8g -jar "$DRIVER_JAR" -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" else echo "Not regenerating data flow instrumented dacapo" fi From 37587d525864758cf75abe7cd308c3416c0b77c1 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 17:44:42 -0500 Subject: [PATCH 10/12] * Fixed deadlock in Instrumenter * added quiet flag back to dacapo script --- integration-tests/runDacapo.sh | 2 +- .../cs/psl/phosphor/driver/Instrumenter.java | 109 ++++++++++-------- 2 files changed, 62 insertions(+), 49 deletions(-) diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index 136291e73..d34aa76e6 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -25,7 +25,7 @@ cd "$BUILD_DIR" if [ ! -d "$INST_DACAPO_DIR" ]; then echo "Creating data flow instrumented dacapo" - java -Xmx8g -jar "$DRIVER_JAR" -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" + java -Xmx8g -jar "$DRIVER_JAR" -q -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" else echo "Not regenerating data flow instrumented dacapo" fi diff --git a/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java index 83170c129..af852281e 100644 --- a/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java +++ b/phosphor-driver/src/main/java/edu/columbia/cs/psl/phosphor/driver/Instrumenter.java @@ -8,7 +8,10 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.*; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; public final class Instrumenter { private final SignatureRemover signatureRemover = new SignatureRemover(); @@ -18,6 +21,7 @@ public final class Instrumenter { private final Instrumentation instrumentation; private final AtomicInteger count = new AtomicInteger(0); private final boolean verbose; + private final Queue> futures = new ConcurrentLinkedQueue<>(); public Instrumenter(Instrumentation instrumentation, boolean verbose) { if (instrumentation == null) { @@ -33,8 +37,7 @@ public void process(File source, File destination) throws IOException, Interrupt } else if (!source.isDirectory() && !isClass(source.getName()) && !isArchive(source.getName())) { throw new IllegalArgumentException("Unknown source file type: " + source); } - Queue> futures = new LinkedList<>(); - processFile(futures, source, destination); + collectFiles(source, destination); while (!futures.isEmpty()) { futures.poll().get(); } @@ -69,17 +72,18 @@ private void instrumentClass(byte[] classFileBuffer, OutputStream output) throws } } - private void processFile(Collection> futures, File source, File destination) - throws IOException, InterruptedException { + private void collectFiles(File source, File destination) throws IOException { if (source.isDirectory()) { InstrumentUtil.ensureDirectory(destination); for (File child : Objects.requireNonNull(source.listFiles())) { - processFile(futures, child, new File(destination, child.getName())); + collectFiles(child, new File(destination, child.getName())); } } else if (isClass(source.getName())) { futures.add(executor.submit(() -> instrumentClass(source, destination), null)); } else if (isArchive(source.getName())) { - processZip(Files.newInputStream(source.toPath()), Files.newOutputStream(destination.toPath())); + byte[] buffer = InstrumentUtil.readAllBytes(source); + OutputStream out = Files.newOutputStream(destination.toPath()); + futures.add(executor.submit(() -> processZip(buffer, out), null)); } else { if (copy(source, destination)) { if (source.canExecute() && !destination.setExecutable(true)) { @@ -95,52 +99,61 @@ private void processFile(Collection> futures, File source, File des } } - private void processZip(InputStream in, OutputStream out) throws IOException, InterruptedException { - try { - List> futures = new LinkedList<>(); - try (ZipInputStream zin = new ZipInputStream(in)) { - for (ZipEntry entry; (entry = zin.getNextEntry()) != null; ) { - ZipEntry finalEntry = entry; - if (entry.isDirectory()) { - futures.add(executor.submit(() -> new ZipResult(finalEntry, null))); - } else if (!signatureRemover.removeEntry(entry.getName())) { - byte[] buffer = InputStreams.readFully(zin); - futures.add(executor.submit(() -> new ZipResult(finalEntry, buffer))); - } + private void finishArchive(List> entries, OutputStream out) { + for (Future future : entries) { + // Submit a new task to finish this archive to prevent blocking + if (!future.isDone()) { + executor.submit(() -> finishArchive(entries, out)); + return; + } + } + try (ZipOutputStream zos = new ZipOutputStream(out)) { + for (Future future : entries) { + ZipResult result = future.get(); + ZipEntry entry = result.entry; + ZipEntry outEntry = new ZipEntry(entry.getName()); + outEntry.setMethod(entry.getMethod()); + if (entry.getMethod() == ZipEntry.STORED) { + // Uncompressed entries require entry size and CRC + outEntry.setSize(result.buffer.length); + outEntry.setCompressedSize(result.buffer.length); + CRC32 crc = new CRC32(); + crc.update(result.buffer, 0, result.buffer.length); + outEntry.setCrc(crc.getValue()); } + zos.putNextEntry(outEntry); + zos.write(result.buffer); + zos.closeEntry(); } - writeZipResults(out, futures); - } catch (IOException e) { + zos.finish(); + } catch (ExecutionException | InterruptedException | IOException e) { errors.add(e); - InstrumentUtil.copy(in, out); } } - private void writeZipResults(OutputStream out, List> futures) - throws IOException, InterruptedException { - try (ZipOutputStream zos = new ZipOutputStream(out)) { - for (Future future : futures) { - try { - ZipResult result = future.get(); - ZipEntry entry = result.entry; - ZipEntry outEntry = new ZipEntry(entry.getName()); - outEntry.setMethod(entry.getMethod()); - if (entry.getMethod() == ZipEntry.STORED) { - // Uncompressed entries require entry size and CRC - outEntry.setSize(result.buffer.length); - outEntry.setCompressedSize(result.buffer.length); - CRC32 crc = new CRC32(); - crc.update(result.buffer, 0, result.buffer.length); - outEntry.setCrc(crc.getValue()); - } - zos.putNextEntry(outEntry); - zos.write(result.buffer); - zos.closeEntry(); - } catch (ExecutionException | ZipException e) { - errors.add(e); + private List> submitZipEntries(byte[] in) throws IOException { + List> entries = new LinkedList<>(); + try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(in))) { + for (ZipEntry entry; (entry = zin.getNextEntry()) != null; ) { + ZipEntry finalEntry = entry; + if (entry.isDirectory()) { + entries.add(executor.submit(() -> new ZipResult(finalEntry, null))); + } else if (!signatureRemover.removeEntry(entry.getName())) { + byte[] buffer = InputStreams.readFully(zin); + entries.add(executor.submit(() -> new ZipResult(finalEntry, buffer))); } } - zos.finish(); + } + futures.addAll(entries); + return entries; + } + + private void processZip(byte[] in, OutputStream out) { + try { + List> entries = submitZipEntries(in); + finishArchive(entries, out); + } catch (IOException e) { + errors.add(e); } } @@ -167,7 +180,7 @@ private class ZipResult { private final ZipEntry entry; private final byte[] buffer; - public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedException { + public ZipResult(ZipEntry entry, byte[] buffer) { this.entry = entry; byte[] tempBuffer = new byte[0]; if (buffer != null) { @@ -178,9 +191,9 @@ public ZipResult(ZipEntry entry, byte[] buffer) throws IOException, InterruptedE if (entry.getName().endsWith(".class")) { instrumentClass(buffer, out); } else if (entry.getName().endsWith(".jar")) { - processZip(in, out); + processZip(buffer, out); } else if (!signatureRemover.filterEntry(entry.getName(), in, out)) { - InstrumentUtil.copy(in, out); + out.write(buffer); } tempBuffer = out.toByteArray(); } catch (IOException | RuntimeException e) { From a37f32b5b98140698587d1e983adf3bfe5ee6811 Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 18:47:07 -0500 Subject: [PATCH 11/12] * Fixed overwrite of embedded configuration options in Phosphor * clarified command synopsis printed by driver * Updated readme --- .../columbia/cs/psl/phosphor/Phosphor.java | 4 +- .../cs/psl/phosphor/PhosphorOption.java | 2 +- README.md | 247 ++++++++++-------- 3 files changed, 142 insertions(+), 111 deletions(-) diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java index 0df15152e..8d13e0a04 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java @@ -19,7 +19,9 @@ public static void initialize(String agentArgs, InstrumentationAdaptor instrumen Phosphor.instrumentation = instrumentation; instrumentation.addTransformer(new ClassSupertypeReadingTransformer()); RUNTIME_INST = true; - PhosphorOption.configure(true, parseOptions(agentArgs)); + if (agentArgs != null) { + PhosphorOption.configure(true, parseOptions(agentArgs)); + } if (System.getProperty("phosphorCacheDirectory") != null) { Configuration.CACHE = TransformationCache.getInstance(System.getProperty("phosphorCacheDirectory")); } diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java index 522fd76fa..ecbb26116 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/PhosphorOption.java @@ -377,7 +377,7 @@ public static Options createOptions(boolean forRuntimeInst) { } public static CommandLine configure(boolean isRuntime, String[] args) { - String commandSynopsis = "java -jar phosphor.jar [OPTIONS] "; + String commandSynopsis = "java -jar phosphor-driver.jar [OPTIONS] "; Options options = createOptions(isRuntime); CommandLine line; try { diff --git a/README.md b/README.md index 7c95d25f0..b28da1e92 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Phosphor: Dynamic Taint Tracking for the JVM -Phosphor is a system for performing dynamic taint analysis in the JVM, on commodity JVMs (e.g. Oracle's HotSpot or -OpenJDK's IcedTea). -Phosphor associates labels, which we referred to as taint tags, with program data and propagates these -labels along information "flows" at runtime. +Phosphor is a system for performing dynamic taint tracking in the Java Virtual Machine (JVM), on commodity JVMs (e.g. +Oracle's HotSpot or OpenJDK's IcedTea). +Phosphor uses Java bytecode instrumentation to associate labels, which we referred to as taint tags, with program data +and propagate these labels along information "flows" at runtime. This repository contains the source code for Phosphor. For more information about how Phosphor works and its uses, please refer to our [OOPSLA 2014 paper](http://jonbell.net/publications/phosphor), -[ISSTA 2015 Tool Demo](http://mice.cs.columbia.edu/getTechreport.php?techreportID=1601) +[ISSTA 2015 Tool Demo](http://mice.cs.columbia.edu/getTechreport.php?techreportID=1601), or email [Jonathan Bell](mailto:jbell@cs.columbia.edu). José Cambronero also maintains a [series of examples on using Phosphor](https://github.com/josepablocam/phosphor-examples/). @@ -32,10 +32,9 @@ Remaining tasks for this branch before promotion: * Implement control tracking semantics (currently entirely unimplemented) * Performance optimization * Improve documentation -* Consider simplifying distribution of the now multiple jars, or alternatively remove the phosphor-distribution module * Consider removing the PHOSPHOR_TAG field from objects, and remove `MultiTainter.taintedObject` -## Building +## Building Phosphor ### Requirements @@ -46,7 +45,7 @@ Remaining tasks for this branch before promotion: 1. Clone or download this repository. 2. Ensure that some version of the JDK 9+ is installed. - A JDK can be downloads from [Oracle](https://www.oracle.com/java/technologies/downloads/) or + A JDK can be downloaded from [Oracle](https://www.oracle.com/java/technologies/downloads/) or the [Adoptium Working Group](https://adoptium.net/temurin/releases/). 3. Set the JAVA_HOME environmental variable to the path of this JDK installation. On Linux and Mac this can be done by running `export JAVA_HOME=`, where <PATH-TO-JDK> is the path @@ -59,136 +58,166 @@ Remaining tasks for this branch before promotion: ## Running Phosphor's Tests Once you have built Phosphor according to the directions described above in the -Section ["Building Phosphor"](#Building-Phosphor), you can run Phosphor's tests and examples. +section ["Building Phosphor"](#Building-Phosphor), you can run Phosphor's tests and examples. Although Phosphor currently requires Java 9+ to build, it can also be used on Java 8. If you would like to run Phosphor's tests on Java 8, build Phosphor using Java 9+, then change the -JAVA_HOME environmental variable to the path of a JDK 8 installation before running the tests. - -TODO what command should you run and what will this command do. -TODO Describe some of the tests - -## Running the DaCapo Benchmarks +`JAVA_HOME` environmental variable to the path of a JDK 8 installation before running the tests. +To run Phosphor the root directory of this project, run `mvn -pl :integration-tests verify`. +The first time you run this command, Maven will invoke the Phosphor Maven plugin to create +Phosphor-instrumented Java installations. +These instrumented Java installation are cached for future use and will not be recreated unless one of the +Phosphor JARs, the configuration used to create them, or the value of `JAVA_HOME` changes. +Once the Phosphor Maven plugin finishes creating the instrumented Java installations the tests will run. +These tests demonstrate how Phosphor can be used and are a good reference when first learning Phosphor. ## Creating an Instrumented Java Installation -In order to associate labels with data and to propagate these labels, Phosphor uses Java bytecode instrumentation to -modify Java classes. In order to track the flow of information through classes in the Java Class Library (JCL), such as `java.lang.String` -and `java.util.List`, Phosphor must also instrument the bytecode of JCL classes. +and `java.util.List`, Phosphor must instrument the bytecode of JCL classes. Therefore, the first step when using Phosphor is to create an instrumented Java installation (i.e., Java Development Kit or Java Runtime Environment). A Java installation can be downloaded from [Oracle](https://www.oracle.com/java/technologies/downloads/) or the [Adoptium Working Group](https://adoptium.net/temurin/releases/). -Once you have obtained a Java installation, it can be instrumented either using Phosphor's [driver JAR](#Driver-Jar) or +Once you have obtained a Java installation, it can be instrumented either using Phosphor's [driver](#Driver) or [Maven plugin](#Maven-Plugin). -We discuss both options and how to configure Phosphor below. - -### Driver JAR - -### Maven Plugin - -### Configuring Phosphor - -## Running an Application with Phosphor - -## Extending Phosphor - -## Running - -Phosphor works by modifying your application's bytecode to perform data flow tracking. -To be complete, Phosphor also modifies the bytecode of JRE-provided classes, too. -The first step to using Phosphor is generating an instrumented version of your runtime environment. -We have tested Phosphor with Oracle and OpenJDK Java 8 runtimes. - -The instrumenter takes two primary arguments: first a path containing the classes to instrument, and then a destination -for the instrumented classes. -You can also specify to track taint tags through control flow, to use objects as tags ( -instead of integers), or to automatically perform taint marking in particular methods using the various options as shown -by invoking Phosphor with the "-help" option. +We discuss both options below. + +**Important note on OpenJDK vs Oracle's Java installations:** +Oracle's Java installations requires that the JAR that contains the cryptography routines `jce.jar` be signed by +Oracle for export control purposes. +OpenJDK does not. +Phosphor instrumentation will break these signatures. +Therefore, it is not possible to use Phosphor with Oracle's Java installation *and* use the cryptography functionality. + +### Driver + +The Phosphor driver can be used apply Phosphor instrumentation to Java classes in a Java installation, +directory, or archive. +If you have built Phosphor according to the directions described above in the +section ["Building Phosphor"](#Building-Phosphor), then the driver JAR will be available at +`phosphor-driver/target/phosphor-driver-0.1.0-SNAPSHOT.jar` relative to the root of this project. +The latest snapshot of the driver JAR is available at the +[Sonatype OSS Repository Hosting (OSSRH)](https://oss.sonatype.org/content/repositories/snapshots/edu/gmu/swe/phosphor/). +The driver JAR can also be acquired using the Maven dependency: ``` -usage: java -jar phosphor.jar [OPTIONS] [input] [output] - -controlTrack Enable taint tracking through control flow - -help print this message - -withoutDataTrack Disable taint tracking through data flow - (on by default) + + ... + + + edu.gmu.swe.phosphor + phosphor-driver + VERSION + + + ... + ``` -Phosphor now should be configured to correctly run JUnit tests (with taint tracking) in most environments (Mac + Linux + -Windows). -Running `mvn verify` should cause Phosphor to generate several different instrumented JRE's (for multitaint -use, int-tag taint use, and control track use) into the project's `target` directory, then run unit tests in that JRE -that are automatically tracked. -You take a look at the test cases to see some example usage. -Tests that end -in `ObjTagITCase` are executed with Phosphor configured for object tags (multi tainting), and `ImplicitITCase` tests run -in the control tracking mode. +The driver supports archives of types JAR, WAR, ZIP, and JMOD. +The driver takes a list of options followed by a source location and a destination location: -### Instrumenting a Java 9+ VM +``` +java -jar [OPTIONS] +``` -We'll assume that in all of the code examples below, we're in the same directory as this README.md, that you have -already run `mvn package`, and that the JVM is located -here: `/Library/Java/JavaVirtualMachines/adoptopenjdk-16.jdk/Contents/Home` (modify this path in the commands below to -match your environment). +The driver will create a Phosphor-instrumented copy of Java installation, directory, or archive located at the +specified source location at place it at the specified destination location. +The options you specify allow you to control how Phosphor instruments the specified source. +A detailed list of available options can be accessed by running: -For Java >= 9, we generate an instrumented JVM by using the `jlink` tool. We have created a jar ( -in `phosphor-instrument-jigsaw`) that wraps this entire process, invoking these tools automatically. To instrument the -JVM, run this command: -`java -jar phosphor-instrument-jigsaw/target/phosphor-instrument-jigsaw-0.1.0-SNAPSHOT.jar /Library/Java/JavaVirtualMachines/adoptopenjdk-16.jdk/Contents/Home jre-inst` +``` +java -jar -help +``` -The next step is to instrument the code which you would like to track. This time, we will use Phosphor's normal -instrumenter (not the jigsaw JVM instrumenter), and will pass your entire (compiled) code base to Phosphor for -instrumentation, and specify an output folder for that. -Example: `java -jar Phosphor/target/Phosphor-0.1.0-SNAPSHOT.jar path-to-application output-path-for-instrumented-code` +### Maven Plugin -We can now run the instrumented code using our instrumented JRE, as such: +To create a Phosphor-instrumented Java installation as part of your Maven build, add the +`phosphor-instrument-maven-plugin` in your pom: ``` -jre-inst/bin/java -javaagent:phosphor-jigsaw-javaagent/target/phosphor-jigsaw-javaagent-0.1.0-SNAPSHOT.jar -cp path-to-instrumented-code your.main.class -```` + + ... + + ... + + edu.gmu.swe.phosphor + phosphor-instrument-maven-plugin + VERSION + + ... + + ... + +``` -### Instrumenting a Java 8 VM +See the documentation for the +[InstrumentingMojo](phosphor-instrument-maven-plugin/src/main/java/edu/columbia/cs/psl/phosphor/plugin/InstrumentMojo.java) +for more detailing on how to configure and use the Phosphor Maven plugin. + +## Instrumenting Your Application + +If you choose, you can also use the Phosphor driver to instrument you application classes before running your +application. +This step is optional; +Phosphor will dynamically instrument any classes not already instrumented at runtime as they are loaded by the JVM. +If you plan to run these classes in a Java 8 JVM, you **must** add `-java8` to the list of options passed to the driver. +If you want to Phosphor to cache classes that are dynamically instrumented, then you can add the Java option +`-DphosphorCacheDirectory=` when running your application, where <CACHE-DIRECTORY> is the file +path to the directory where Phosphor should store the cached instrumented class files. + +## Running Your Application with Phosphor + +Once you have created an instrumented Java installation (see the section +["Creating an Instrumented Java Installation"](#Creating-an-Instrumented-Java-Installation), you can run your +application with Phosphor. +Locate the JAR for Phosphor's Java agent. +If you have built Phosphor according to the directions described above in the +section ["Building Phosphor"](#Building-Phosphor), then the agent JAR will be available at +`Phosphor/target/Phosphor-0.1.0-SNAPSHOT.jar` relative to the root of this project. +The latest snapshot of the agent JAR is available at the +[Sonatype OSS Repository Hosting (OSSRH)](https://oss.sonatype.org/content/repositories/snapshots/edu/gmu/swe/phosphor/). +The agent JAR can also be acquired using the Maven dependency: -We'll assume that in all of the code examples below, we're in the same directory (which has a copy of -Phosphor-0.0.5-SNAPSHOT.jar, which you generated by downloading Phosphor, and running `mvn package`), and that the JRE -is located here: `/Library/Java/JavaVirtualMachines/jdk1.8.0_222.jdk/Contents/Home` (modify this path in the commands -below to match your environment). +``` + + ... + + + edu.gmu.swe.phosphor + Phosphor + VERSION + + + ... + +``` -*Important note on OpenJDK vs Oracle's JDK:* Oracle's JVM requires that the jar that contains all of the cryptography -routines (`jce.jar`) be signed by Oracle, for export control purposes. OpenJDK does not. When Phosphor instruments the -JVM, it will break these signatures. Hence, it is not possible to use Phosphor with Oracle's JDK *and* use the -cryptography functionality of the JVM without some extra work. The first option is (if you have been granted a signing -key by Oracle), you can sign the resulting jar. The more attainable option is to instead use the OpenJDK `jce.jar` file -with your Oracle JVM, which will not have the signature checks, and will work just fine. Alternatively, if this is too -complicated (and you need cryptography support), you can just choose to use OpenJDK entirely (which does not have this -check). +Once you have located the JAR for Phosphor's Java agent, execute the same Java command you would normally to run +your application using the Java executable inside the instrumented Java installation you created and +add `javaagent` and `bootclasspath` Java options for Phosphor. +If running java using the `-jar` option run: -Then, to instrument the JRE we'll run: -`java -jar Phosphor-0.1.0-SNAPSHOT.jar /Library/Java/JavaVirtualMachines/jdk1.8.0_222.jdk/Contents/Home jre-inst` +``` +/bin/java \ + -Xbootclasspath/a: \ + -javaagent: \ + [ options ] -jar file.jar [ argument ... ] +``` -After you do this, make sure to chmod +x the binaries in the new folder, e.g. `chmod +x jre-inst/bin/*` +Otherwise run: -The next step is to instrument the code which you would like to track. This time when you run the instrumenter, pass -your entire (compiled) code base to Phosphor for instrumentation, and specify an output folder for that. +``` +/bin/java \ + -Xbootclasspath/a: \ + -javaagent: \ + [options] class [ argument ... ] +``` -We can now run the instrumented code using our instrumented JRE, as such: +Where: -``` -export JAVA_HOME=jre-inst/ -$JAVA_HOME/bin/java -Xbootclasspath/a:Phosphor-0.1.0-SNAPSHOT.jar -javaagent:Phosphor-0.1.0-SNAPSHOT.jar -cp path-to-instrumented-code your.main.class -```` - -Note: It is not 100% necessary to instrument your application/library code in advance - the javaagent will detect any -uninstrumented class files as they are being loaded into the JVM and instrument them as necessary. If you want to do -this, then you may want to add the flag `-javaagent:Phosphor-0.1.0-SNAPSHOT.jar=cacheDir=someCacheFolder` and Phosphor -will cache the generated files in `someCacheFolder` so they aren't regenerated every run. If you take a look at the -execution of Phosphor's JUnit tests, you'll notice that this is how they are instrumented. It's always necessary to -instrument the JRE in advance though for bootstrapping. - -New 2/27/19: You can no longer specify auto taint methods (what were sources/sinks/taint through methods) for the static -instrumenter. Instead, ALL autotaint instrumentation happens via the java agent (this makes it possible to detect -child-classes of auto taint classes). You can specify the files to the java agent using the -syntax `-javaagent:Phosphor-0.0.4-SNAPSHOT.jar=taintSources={taintSourceFile},taintSinks={taintSinksFile},taintThrough={taintThroughFile}` +* <> is the instrumented Java installation you created. +* <> is the path to the JAR for Phosphor's Java agent. ## Interacting with Phosphor @@ -218,7 +247,7 @@ functionality. If you are interested in helping, please contact us. ## Contact -Please email [Jonathan Bell](mailto:bellj@gmu.edu) with any feedback or questions. +Please email [Jonathan Bell](mailto:bellj@gmu.edu) with comments, suggestions, or questions. This project is still under development and we welcome any feedback. ## License From c3d97e302b5b4259ab71f8435a76bca1324d7e5b Mon Sep 17 00:00:00 2001 From: katherine-hough <32645020+katherine-hough@users.noreply.github.com> Date: Sun, 26 Nov 2023 19:16:22 -0500 Subject: [PATCH 12/12] * Fixed issue with DaCapo on Java 8 --- .../java/edu/columbia/cs/psl/phosphor/Phosphor.java | 2 +- integration-tests/pom.xml | 11 +++++++++++ integration-tests/runDacapo.sh | 8 +++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java index 8d13e0a04..38c034e6a 100644 --- a/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java +++ b/Phosphor/src/main/java/edu/columbia/cs/psl/phosphor/Phosphor.java @@ -19,7 +19,7 @@ public static void initialize(String agentArgs, InstrumentationAdaptor instrumen Phosphor.instrumentation = instrumentation; instrumentation.addTransformer(new ClassSupertypeReadingTransformer()); RUNTIME_INST = true; - if (agentArgs != null) { + if (agentArgs != null || Configuration.IS_JAVA_8) { PhosphorOption.configure(true, parseOptions(agentArgs)); } if (System.getProperty("phosphorCacheDirectory") != null) { diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index fef0f7c5e..46b9f620a 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -19,6 +19,7 @@ false ${edu.gmu.swe.phosphor:Phosphor:jar} ${edu.gmu.swe.phosphor:phosphor-driver:jar} + false @@ -107,6 +108,15 @@ + + java8 + + 1.8 + + + true + + dacapo @@ -138,6 +148,7 @@ ${phosphor.jar} ${phosphor.driver.jar} ${data.flow.java}/bin/java + ${isJava8} diff --git a/integration-tests/runDacapo.sh b/integration-tests/runDacapo.sh index d34aa76e6..9e2e030a3 100755 --- a/integration-tests/runDacapo.sh +++ b/integration-tests/runDacapo.sh @@ -4,6 +4,7 @@ readonly BUILD_DIR=$1 readonly PHOSPHOR_JAR=$2 readonly DRIVER_JAR=$3 readonly INST_JVM=$4 +readonly IS_JAVA_8=$5 readonly DACAPO_DIR=$BUILD_DIR/dacapo readonly INST_DACAPO_DIR=$BUILD_DIR/dacapo-inst readonly BENCHMARKS=(avrora fop h2 jython luindex pmd sunflow xalan) @@ -23,9 +24,14 @@ if [ ! -d "$DACAPO_DIR" ]; then fi cd "$BUILD_DIR" +phosphor_args="" +if [ "$IS_JAVA_8" == "true" ]; then + phosphor_args="-java8" +fi + if [ ! -d "$INST_DACAPO_DIR" ]; then echo "Creating data flow instrumented dacapo" - java -Xmx8g -jar "$DRIVER_JAR" -q -forceUnboxAcmpEq -withEnumsByValue "$DACAPO_DIR" "$INST_DACAPO_DIR" + java -Xmx8g -jar "$DRIVER_JAR" -q -forceUnboxAcmpEq -withEnumsByValue $phosphor_args "$DACAPO_DIR" "$INST_DACAPO_DIR" else echo "Not regenerating data flow instrumented dacapo" fi