Support dynamic class loading

Support dynamic class loading for native image

Author: ziyilin

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java

@@ -27,6 +27,7 @@
 import static com.oracle.svm.core.util.VMError.guarantee;
 import static com.oracle.svm.jni.JNIObjectHandles.nullHandle;
 import static com.oracle.svm.jvmtiagentbase.Support.callObjectMethod;
+import static com.oracle.svm.jvmtiagentbase.Support.callObjectMethodL;
 import static com.oracle.svm.jvmtiagentbase.Support.check;
 import static com.oracle.svm.jvmtiagentbase.Support.checkJni;
 import static com.oracle.svm.jvmtiagentbase.Support.checkNoException;
@@ -39,6 +40,7 @@
 import static com.oracle.svm.jvmtiagentbase.Support.getClassNameOrNull;
 import static com.oracle.svm.jvmtiagentbase.Support.getDirectCallerClass;
 import static com.oracle.svm.jvmtiagentbase.Support.getMethodDeclaringClass;
+import static com.oracle.svm.jvmtiagentbase.Support.getMethodFullNameAtFrame;
 import static com.oracle.svm.jvmtiagentbase.Support.getObjectArgument;
 import static com.oracle.svm.jvmtiagentbase.Support.handleException;
 import static com.oracle.svm.jvmtiagentbase.Support.jniFunctions;
@@ -52,18 +54,22 @@
 import static com.oracle.svm.jvmtiagentbase.jvmti.JvmtiEvent.JVMTI_EVENT_NATIVE_METHOD_BIND;
 import static org.graalvm.word.WordFactory.nullPointer;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.graalvm.compiler.core.common.NumUtil;
+import org.graalvm.compiler.phases.common.LazyValue;
 import org.graalvm.nativeimage.StackValue;
 import org.graalvm.nativeimage.UnmanagedMemory;
 import org.graalvm.nativeimage.c.function.CEntryPoint;
@@ -80,6 +86,7 @@
 import org.graalvm.nativeimage.c.type.WordPointer;
 import org.graalvm.word.WordFactory;
 
+import com.oracle.svm.configure.trace.AccessAdvisor;
 import com.oracle.svm.core.c.function.CEntryPointOptions;
 import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.jni.JNIObjectHandles;
@@ -126,7 +133,8 @@ final class BreakpointInterceptor {
     private static NativeImageAgent agent;
 
     private static Map<Long, Breakpoint> installedBreakpoints;
-
+    private static Set<String> definedClasses = ConcurrentHashMap.newKeySet();
+    private static List<String> unsupportedExceptions = new ArrayList<>();
     /**
      * A map from {@link JNIMethodId} to entry point addresses for bound Java {@code native}
      * methods, NOT considering our intercepting functions, i.e., these are the original entry
@@ -169,6 +177,18 @@ private static void traceBreakpoint(JNIEnvironment env, JNIObjectHandle clazz, J
         }
     }
 
+    static void traceBreakpoint(JNIEnvironment env, JNIObjectHandle clazz, JNIObjectHandle declaringClass,
+                    JNIObjectHandle callerClass, String function, boolean allowWrite, boolean unsafeAccess, Object result,
+                    String fieldName) {
+        if (traceWriter != null) {
+            traceWriter.traceCall("reflect", function, getClassNameOr(env, clazz, null, TraceWriter.UNKNOWN_VALUE),
+                            getClassNameOr(env, declaringClass, null, TraceWriter.UNKNOWN_VALUE),
+                            getClassNameOr(env, callerClass, null, TraceWriter.UNKNOWN_VALUE), result, allowWrite, unsafeAccess,
+                            fieldName);
+            guarantee(!testException(env));
+        }
+    }
+
     private static boolean forName(JNIEnvironment jni, Breakpoint bp) {
         JNIObjectHandle callerClass = getDirectCallerClass();
         JNIObjectHandle name = getObjectArgument(0);
@@ -573,6 +593,96 @@ private static boolean handleGetSystemResources(JNIEnvironment jni, Breakpoint b
         return true;
     }
 
+    /**
+     * java.lang.ClassLoader.postDefineClass is always called in java.lang.ClassLoader.defineClass,
+     * so intercepting postDefineClass is equivalent to intercepting defineClass but with extra
+     * benefit of being always able to get defined class' name even if defineClass' classname
+     * parameter is null.
+     */
+    @SuppressWarnings("unused")
+    private static boolean postDefineClass(JNIEnvironment jni, Breakpoint bp) {
+        boolean isDynamicallyGenerated = false;
+
+        // Get class name from the argument "name" of
+        // defineClass(String name, byte[] b, int off, int len)
+        // The first argument is implicitly "this", so "name" is the 2nd parameter.
+        String nameFromDefineClassParam = fromJniString(jni, getObjectArgument(1, 1));
+        final String definedClassName;
+        JNIObjectHandle self = getObjectArgument(0);
+        // 1. Don't have a name for class before defining.
+        // The class is dynamically generated.
+        if (nameFromDefineClassParam == null) {
+            isDynamicallyGenerated = true;
+            // Get name from parameter "c" of method postDefineClass(Class<?> c, ProtectionDomain
+            // pd)
+            definedClassName = getClassNameOrNull(jni, getObjectArgument(1));
+        } else {
+            definedClassName = nameFromDefineClassParam;
+            // Filter out internal classes which are definitely not dynamically generated
+            // CallerClass is always java.lang.ClassLoader, we only check the defined class
+            AccessAdvisor postDefineCLassAccessAdvisor = new AccessAdvisor();
+            postDefineCLassAccessAdvisor.setInLivePhase(true);
+            if (postDefineCLassAccessAdvisor.shouldIgnore(new LazyValue<>(() -> definedClassName), new LazyValue<>(() -> null))) {
+                isDynamicallyGenerated = false;
+            }
+
+            // 2. Class with name starts with $ or contains $$ is usually dynamically generated
+            String className = definedClassName.substring(definedClassName.lastIndexOf('.') + 1);
+            if (className.startsWith("$") || className.contains("$$")) {
+                isDynamicallyGenerated = true;
+            } else {
+                // 3. A dynamically defined class always return null
+                // when call java.lang.ClassLoader.getResource(classname)
+                // This is the accurate but slow way.
+                String asResourceName = definedClassName.replace('.', '/') + ".class";
+                try (CCharPointerHolder resourceNameHolder = toCString(asResourceName);) {
+                    JNIObjectHandle resourceNameJString = jniFunctions().getNewStringUTF().invoke(jni, resourceNameHolder.get());
+                    JNIObjectHandle returnValue = callObjectMethodL(jni, self, agent.handles().javaLangClassLoaderGetResource, resourceNameJString);
+                    isDynamicallyGenerated = returnValue.equal(nullHandle());
+                }
+            }
+        }
+
+        // CallerClass is always java.lang.ClassLoader, we only check the defined class
+        Object result = false;
+        boolean justAdded = definedClasses.add(definedClassName);
+        if (isDynamicallyGenerated) {
+            if (!justAdded) {
+                unsupportedExceptions.add("Class " + definedClassName + " has been defined before. Multiple definitions are not supported.\n" +
+                                ClassLoaderDefineClassSupport.getStackTrace(jni).toString());
+                return true;
+            }
+            // Check the caller is using byte array or directedBuffer
+            String caller = getMethodFullNameAtFrame(jni, 1);
+            boolean isByteArray = "java.lang.ClassLoader.defineClass(Ljava/lang/String;[BIILjava/security/ProtectionDomain;)Ljava/lang/Class;".equals(caller);
+            // Verify if defineClass succeeds
+            // As we hook on postDefineClass method which is the last step of defineClass, so if
+            // it can execute successfully, the whole defineClass method can execute
+            // successfully
+            JNIValue args = StackValue.get(2, JNIValue.class);
+            args.addressOf(0).setObject(getObjectArgument(1));
+            args.addressOf(1).setObject(getObjectArgument(2));
+            jniFunctions().getCallVoidMethodA().invoke(jni, self, bp.method, args);
+            if (clearException(jni)) {
+                // No need to proceed if any exception happens
+                result = false;
+            } else {
+                result = true;
+            }
+            try {
+                JNIObjectHandle callerClass = getDirectCallerClass();
+                ClassLoaderDefineClassSupport dynamicSupport = new ClassLoaderDefineClassSupport(jni, callerClass,
+                                definedClassName, traceWriter, agent, isByteArray);
+                dynamicSupport.trace(result);
+                return true;
+            } catch (NoSuchAlgorithmException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            return true;
+        }
+    }
+
     private static boolean newProxyInstance(JNIEnvironment jni, Breakpoint bp) {
         JNIObjectHandle callerClass = getDirectCallerClass();
         JNIObjectHandle classLoader = getObjectArgument(0);
@@ -1269,15 +1379,29 @@ private static void bindNativeBreakpoint(JNIEnvironment jni, NativeBreakpoint bp
         }
     }
 
+    public static void reportExceptions() {
+        if (!unsupportedExceptions.isEmpty()) {
+            System.err.println(unsupportedExceptions.size() + " unsupported features are detected ");
+            StringBuilder errorMsg = new StringBuilder();
+            for (int i = 0; i < unsupportedExceptions.size(); i++) {
+                errorMsg.append(unsupportedExceptions.get(i)).append("\n");
+            }
+            throw new UnsupportedOperationException(errorMsg.toString());
+        } else {
+            unsupportedExceptions = null;
+        }
+    }
+
     public static void onUnload() {
         installedBreakpoints = null;
         nativeBreakpoints = null;
         observedExplicitLoadClassCallSites = null;
         traceWriter = null;
+        definedClasses = null;
     }
 
     private interface BreakpointHandler {
-        boolean dispatch(JNIEnvironment jni, Breakpoint bp);
+        boolean dispatch(JNIEnvironment jni, Breakpoint bp) throws IOException;
     }
 
     private static final BreakpointSpecification[] BREAKPOINT_SPECIFICATIONS = {
@@ -1319,6 +1443,10 @@ private interface BreakpointHandler {
                     brk("java/lang/reflect/Proxy", "getProxyClass", "(Ljava/lang/ClassLoader;[Ljava/lang/Class;)Ljava/lang/Class;", BreakpointInterceptor::getProxyClass),
                     brk("java/lang/reflect/Proxy", "newProxyInstance",
                                     "(Ljava/lang/ClassLoader;[Ljava/lang/Class;Ljava/lang/reflect/InvocationHandler;)Ljava/lang/Object;", BreakpointInterceptor::newProxyInstance),
+                    /*
+                     * For dumping dynamically generated classes
+                     */
+                    brk("java/lang/ClassLoader", "postDefineClass", "(Ljava/lang/Class;Ljava/security/ProtectionDomain;)V", BreakpointInterceptor::postDefineClass),
 
                     brk("java/io/ObjectStreamClass", "<init>", "(Ljava/lang/Class;)V", BreakpointInterceptor::objectStreamClassConstructor),
                     optionalBrk("java/util/ResourceBundle",

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/ClassLoaderDefineClassSupport.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Alibaba Group Holding Limited. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.agent;
+
+import com.oracle.svm.core.util.JavaClassUtil;
+import com.oracle.svm.jni.nativeapi.JNIEnvironment;
+import com.oracle.svm.jni.nativeapi.JNIMethodId;
+import com.oracle.svm.jni.nativeapi.JNIObjectHandle;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CTypeConversion;
+import org.graalvm.nativeimage.c.type.VoidPointer;
+import org.graalvm.word.WordFactory;
+
+import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
+
+import static com.oracle.svm.jvmtiagentbase.Support.getIntArgument;
+import static com.oracle.svm.jvmtiagentbase.Support.getMethodFullNameAtFrame;
+import static com.oracle.svm.jvmtiagentbase.Support.getObjectArgument;
+import static com.oracle.svm.jvmtiagentbase.Support.jniFunctions;
+
+/**
+ * Support dynamic class loading that is implemented by java.lang.ClassLoader.defineClass.
+ */
+public class ClassLoaderDefineClassSupport {
+
+    private static final String DEFINE_CLASS_BYTEARRAY_SIG = "java.lang.ClassLoader.defineClass(Ljava/lang/String;[BII)Ljava/lang/Class;";
+    protected JNIEnvironment jni;
+    protected JNIObjectHandle callerClass;
+    protected final String generatedClassName;
+    protected TraceWriter traceWriter;
+    protected NativeImageAgent agent;
+    protected String generatedClassHashCode = null;
+    protected byte[] values = null;
+    private boolean isByteArray;
+
+    protected String callerMethod;
+
+    public ClassLoaderDefineClassSupport(JNIEnvironment jni, JNIObjectHandle callerClass, String generatedClassName, TraceWriter traceWriter, NativeImageAgent agent, boolean isByteArray) {
+        this.jni = jni;
+        this.callerClass = callerClass;
+        // Make sure use qualified name for generatedClassName
+        this.generatedClassName = generatedClassName.replace('/', '.');
+        this.traceWriter = traceWriter;
+        this.agent = agent;
+        callerMethod = getMethodFullNameAtFrame(jni, 1);
+        this.isByteArray = isByteArray;
+    }
+
+    protected byte[] getClassContentsFromByteArray() {
+        // bytes parameter of defineClass method
+        JNIObjectHandle bytes = getClassDefinition();
+        // len parameter of defineClass method
+        int length = getClassDefinitionBytesLength();
+        // Get generated class' byte array
+        CCharPointer byteArray = jniFunctions().getGetByteArrayElements().invoke(jni, bytes, WordFactory.nullPointer());
+        byte[] contents = new byte[length];
+        try {
+            CTypeConversion.asByteBuffer(byteArray, length).get(contents);
+        } finally {
+            jniFunctions().getReleaseByteArrayElements().invoke(jni, bytes, byteArray, 0);
+        }
+        return contents;
+    }
+
+    protected byte[] getClassContentsFromDirectBuffer() {
+        // DirectBuffer parameter of defineClass
+        JNIObjectHandle directbuffer = getClassDefinition();
+
+        // Get byte array from DirectBuffer
+        VoidPointer baseAddr = jniFunctions().getGetDirectBufferAddress().invoke(jni, directbuffer);
+        JNIMethodId limitMId = agent.handles().getMethodId(jni, agent.handles().javaNioByteBuffer, "limit", "()I", false);
+        int limit = jniFunctions().getCallIntMethod().invoke(jni, directbuffer, limitMId);
+        ByteBuffer classContentsAsByteBuffer = CTypeConversion.asByteBuffer(baseAddr, limit);
+        byte[] contents = new byte[classContentsAsByteBuffer.limit()];
+        classContentsAsByteBuffer.get(contents);
+        classContentsAsByteBuffer.position(0);
+        return contents;
+    }
+
+    private String calculateGeneratedClassSHA() throws NoSuchAlgorithmException {
+        if (generatedClassHashCode == null) {
+            generatedClassHashCode = JavaClassUtil.getSHAWithoutSourceFileInfo(getClassContents());
+        }
+        return generatedClassHashCode;
+    }
+
+    public static StringBuilder getStackTrace(JNIEnvironment jni) {
+        StringBuilder trace = new StringBuilder();
+        int i = 0;
+        int maxDepth = 20;
+        while (i < maxDepth) {
+            String methodName = getMethodFullNameAtFrame(jni, i++);
+            if (methodName == null) {
+                break;
+            }
+            trace.append("    ").append(methodName).append("\n");
+        }
+        if (i >= maxDepth) {
+            trace.append("    ").append("...").append("\n");
+        }
+        return trace;
+    }
+
+    public boolean checkSupported() {
+        return DEFINE_CLASS_BYTEARRAY_SIG.equals(callerMethod);
+    }
+
+    public void trace(Object result) throws NoSuchAlgorithmException {
+        if (values == null) {
+            values = getClassContents();
+        }
+        generatedClassHashCode = calculateGeneratedClassSHA();
+        if (generatedClassName != null && values != null && result != null) {
+            // Trace dynamically generated class in config file
+            traceWriter.traceDefineClass(generatedClassName, generatedClassHashCode, values, result);
+        }
+    }
+
+    public byte[] getClassContents() {
+        assert checkSupported();
+        if (values == null) {
+            if (isByteArray) {
+                values = getClassContentsFromByteArray();
+            } else {
+                values = getClassContentsFromDirectBuffer();
+            }
+
+        }
+        return values;
+    }
+
+    /**
+     * Get value of argument "b" from java.lang.ClassLoader.defineClass(String name, byte[] b, int
+     * off, int len, ProtectionDomain protectionDomain) "b" is the 3rd argument. because the 1st
+     * argument of instance method is always "this"
+     */
+    protected JNIObjectHandle getClassDefinition() {
+        assert checkSupported();
+        return getObjectArgument(1, 2);
+    }
+
+    /**
+     * Get value of argument "len" from java.lang.ClassLoader. defineClass(String name, byte[] b,
+     * int off, int len, ProtectionDomain protectionDomain) "len" is the 5th argument. because the
+     * 1st argument of instance method is always "this"
+     */
+    protected int getClassDefinitionBytesLength() {
+        assert checkSupported();
+        return getIntArgument(1, 4);
+    }
+}