[GR-73925] Support build-time initialization of downcall handles.

PullRequest: graal/23528

Author: fangerer

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/MethodHandlePlugin.java

@@ -41,13 +41,15 @@
 import jdk.graal.compiler.replacements.nodes.MacroInvokable;
 import jdk.graal.compiler.replacements.nodes.MacroNode;
 import jdk.graal.compiler.replacements.nodes.MethodHandleNode;
+import jdk.graal.compiler.replacements.nodes.MethodHandleNode.GraphAdder;
+import jdk.graal.compiler.replacements.nodes.MethodHandleNode.InvokeFactory;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MethodHandleAccessProvider;
 import jdk.vm.ci.meta.MethodHandleAccessProvider.IntrinsicMethod;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 public class MethodHandlePlugin implements NodePlugin {
-    private final MethodHandleAccessProvider methodHandleAccess;
+    protected final MethodHandleAccessProvider methodHandleAccess;
     private final boolean safeForDeoptimization;
 
     public MethodHandlePlugin(MethodHandleAccessProvider methodHandleAccess, boolean safeForDeoptimization) {
@@ -75,11 +77,7 @@ protected void onCreateHook(MacroInvokable methodHandleNode, GraphBuilderContext
     @Override
     public boolean handleInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) {
         IntrinsicMethod intrinsicMethod = methodHandleAccess.lookupMethodHandleIntrinsic(method);
-        // We skip intrinsification for LINK_TO_NATIVE, because:
-        // 1. HotSpot generates compiler entry jumping to the native wrapper of the target c method.
-        // 2. SVM intrinsification is not yet implemented.
-        // Use String comparison for JDK21 compatibility.
-        if (intrinsicMethod != null && !"LINK_TO_NATIVE".equals(intrinsicMethod.name())) {
+        if (intrinsicMethod != null && canHandleIntrinsicMethod(intrinsicMethod)) {
             InvokeKind invokeKind = b.getInvokeKind();
             if (invokeKind != InvokeKind.Static) {
                 args[0] = b.nullCheckedValue(args[0]);
@@ -91,7 +89,7 @@ public <T extends ValueNode> T add(T node) {
                     return b.add(node);
                 }
             };
-            Invoke invoke = MethodHandleNode.tryResolveTargetInvoke(adder, this::createInvoke, methodHandleAccess, intrinsicMethod, method, b.bci(), invokeReturnStamp, args);
+            Invoke invoke = tryResolveTargetInvoke(adder, this::createInvoke, intrinsicMethod, method, b.bci(), invokeReturnStamp, args);
             if (invoke == null) {
                 MacroInvokable methodHandleNode = createMethodHandleNode(b, method, args, intrinsicMethod, invokeKind, invokeReturnStamp);
                 onCreateHook(methodHandleNode, b);
@@ -159,4 +157,18 @@ public <T extends ValueNode> T add(T node) {
         }
         return false;
     }
+
+    protected boolean canHandleIntrinsicMethod(IntrinsicMethod intrinsicMethod) {
+        assert intrinsicMethod != null;
+        /*
+         * We skip intrinsification for LINK_TO_NATIVE because HotSpot generates compiler entry code
+         * jumping to the native wrapper of the target C method.
+         */
+        return intrinsicMethod != IntrinsicMethod.LINK_TO_NATIVE;
+    }
+
+    protected <T extends Invoke> T tryResolveTargetInvoke(GraphAdder adder, InvokeFactory<T> factory, IntrinsicMethod intrinsicMethod,
+                    ResolvedJavaMethod original, int bci, StampPair returnStamp, ValueNode... arguments) {
+        return MethodHandleNode.tryResolveTargetInvoke(adder, factory, methodHandleAccess, intrinsicMethod, original, bci, returnStamp, arguments);
+    }
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/MethodHandleWithExceptionPlugin.java

@@ -34,7 +34,6 @@
 import jdk.graal.compiler.replacements.nodes.MacroInvokable;
 import jdk.graal.compiler.replacements.nodes.MacroNode;
 import jdk.graal.compiler.replacements.nodes.MethodHandleWithExceptionNode;
-
 import jdk.vm.ci.meta.MethodHandleAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/MethodHandleNode.java

@@ -302,7 +302,7 @@ private static <T extends Invoke> T getTargetInvokeNode(GraphAdder adder, Invoke
      * @param index of the argument to be cast
      * @param type the type the argument should be cast to
      */
-    private static void maybeCastArgument(GraphAdder adder, ValueNode[] arguments, int index, JavaType type) {
+    public static void maybeCastArgument(GraphAdder adder, ValueNode[] arguments, int index, JavaType type) {
         ValueNode argument = arguments[index];
         if (type instanceof ResolvedJavaType && !((ResolvedJavaType) type).isJavaLangObject()) {
             Assumptions assumptions = adder.getAssumptions();

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/MethodHandleWithExceptionNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,29 +31,35 @@
 
 import jdk.graal.compiler.graph.NodeClass;
 import jdk.graal.compiler.nodeinfo.NodeInfo;
+import jdk.graal.compiler.nodes.Invoke;
 import jdk.graal.compiler.nodes.InvokeWithExceptionNode;
 import jdk.graal.compiler.nodes.ValueNode;
 import jdk.graal.compiler.nodes.WithExceptionNode;
 import jdk.graal.compiler.nodes.spi.Simplifiable;
 import jdk.graal.compiler.nodes.spi.SimplifierTool;
-
+import jdk.graal.compiler.replacements.nodes.MethodHandleNode.GraphAdder;
+import jdk.graal.compiler.replacements.nodes.MethodHandleNode.InvokeFactory;
 import jdk.vm.ci.meta.MethodHandleAccessProvider;
 import jdk.vm.ci.meta.MethodHandleAccessProvider.IntrinsicMethod;
 
 /**
  * Node for invocation methods defined on the class {@link MethodHandle}.
  */
 @NodeInfo(cycles = CYCLES_UNKNOWN, cyclesRationale = "see MacroNode", size = SIZE_UNKNOWN, sizeRationale = "see MacroNode")
-public final class MethodHandleWithExceptionNode extends MacroWithExceptionNode implements Simplifiable {
+public class MethodHandleWithExceptionNode extends MacroWithExceptionNode implements Simplifiable {
     public static final NodeClass<MethodHandleWithExceptionNode> TYPE = NodeClass.create(MethodHandleWithExceptionNode.class);
 
     protected final IntrinsicMethod intrinsicMethod;
 
-    public MethodHandleWithExceptionNode(IntrinsicMethod intrinsicMethod, MacroNode.MacroParams p) {
-        super(TYPE, p);
+    protected MethodHandleWithExceptionNode(NodeClass<? extends MacroWithExceptionNode> c, IntrinsicMethod intrinsicMethod, MacroNode.MacroParams p) {
+        super(c, p);
         this.intrinsicMethod = intrinsicMethod;
     }
 
+    public MethodHandleWithExceptionNode(IntrinsicMethod intrinsicMethod, MacroNode.MacroParams p) {
+        this(TYPE, intrinsicMethod, p);
+    }
+
     @Override
     public void simplify(SimplifierTool tool) {
         MethodHandleAccessProvider methodHandleAccess = tool.getConstantReflection().getMethodHandleAccess();
@@ -69,7 +75,7 @@ public WithExceptionNode trySimplify(MethodHandleAccessProvider methodHandleAcce
             invoke.setStamp(stmp);
             return invoke;
         };
-        InvokeWithExceptionNode invoke = MethodHandleNode.tryResolveTargetInvoke(adder, invokeFactory, methodHandleAccess, intrinsicMethod, targetMethod, bci, returnStamp, argumentsArray);
+        InvokeWithExceptionNode invoke = tryResolveTargetInvoke(adder, invokeFactory, methodHandleAccess, argumentsArray);
         if (invoke == null) {
             return this;
         }
@@ -80,4 +86,9 @@ public WithExceptionNode trySimplify(MethodHandleAccessProvider methodHandleAcce
         graph().replaceWithExceptionSplit(this, invoke);
         return invoke;
     }
+
+    /** {@link MethodHandleNode#tryResolveTargetInvoke}. */
+    protected <T extends Invoke> T tryResolveTargetInvoke(GraphAdder adder, InvokeFactory<T> factory, MethodHandleAccessProvider methodHandleAccess, ValueNode... argumentsArray) {
+        return MethodHandleNode.tryResolveTargetInvoke(adder, factory, methodHandleAccess, intrinsicMethod, targetMethod, bci, returnStamp, argumentsArray);
+    }
 }

substratevm/mx.substratevm/suite.py

@@ -839,6 +839,7 @@
                     "jdk.internal.foreign.layout",
                     "jdk.internal.loader",
                     "jdk.internal.reflect",
+                    "jdk.internal.vm.annotation",
                 ],
                 "jdk.internal.vm.ci" : [
                     "jdk.vm.ci.aarch64",
@@ -2843,7 +2844,8 @@
                     "org.graalvm.collections",
                 ],
                 "exports" : [
-                    "* to org.graalvm.nativeimage.builder"
+                    "* to org.graalvm.nativeimage.builder",
+                    "com.oracle.svm.hosted.foreign to jdk.graal.compiler"
                 ],
                 "requiresConcealed": {
                     "jdk.internal.vm.ci" : [

substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,7 @@
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
 import com.oracle.svm.core.util.ImageHeapMap;
+import com.oracle.svm.shared.AlwaysInline;
 import com.oracle.svm.shared.Uninterruptible;
 import com.oracle.svm.shared.singletons.traits.BuiltinTraits.AllAccess;
 import com.oracle.svm.shared.singletons.traits.BuiltinTraits.NoLayeredCallbacks;
@@ -102,6 +103,7 @@ public static ForeignFunctionsRuntime singleton() {
     private final AbiUtils.TrampolineTemplate trampolineTemplate;
 
     private final EconomicMap<NativeEntryPointInfo, FunctionPointerHolder> downcallStubs = ImageHeapMap.create("downcallStubs");
+    private final EconomicMap<MethodType, FunctionPointerHolder> downcallStubInvokers = ImageHeapMap.create("downcallStubInvokers");
     private final EconomicMap<Pair<DirectMethodHandleDesc, JavaEntryPointInfo>, FunctionPointerHolder> directUpcallStubs = ImageHeapMap.create("directUpcallStubs");
     private final EconomicMap<JavaEntryPointInfo, FunctionPointerHolder> upcallStubs = ImageHeapMap.create("upcallStubs");
     private final EconomicSet<ResolvedJavaType> neverAccessesSharedArenaTypes = EconomicSet.create();
@@ -161,6 +163,11 @@ public boolean downcallStubExists(NativeEntryPointInfo nep) {
         return downcallStubs.containsKey(nep);
     }
 
+    @Platforms(Platform.HOSTED_ONLY.class)
+    public boolean downcallStubInvokerExists(MethodType methodType) {
+        return downcallStubInvokers.containsKey(methodType);
+    }
+
     @Platforms(Platform.HOSTED_ONLY.class)
     public int getDowncallStubsCount() {
         return downcallStubs.size();
@@ -171,6 +178,11 @@ public boolean upcallStubExists(JavaEntryPointInfo jep) {
         return upcallStubs.containsKey(jep);
     }
 
+    @Platforms(Platform.HOSTED_ONLY.class)
+    public boolean addDowncallStubInvokerPointer(MethodType methodType, CFunctionPointer ptr) {
+        return downcallStubInvokers.putIfAbsent(methodType, new FunctionPointerHolder(ptr)) == null;
+    }
+
     @Platforms(Platform.HOSTED_ONLY.class)
     public int getUpcallStubsCount() {
         return upcallStubs.size();
@@ -212,14 +224,22 @@ public void registerSafeArenaAccessorMethod(ResolvedJavaMethod method) {
         neverAccessesSharedArenaMethods.add(method);
     }
 
-    CFunctionPointer getDowncallStubPointer(NativeEntryPointInfo nep) {
+    public CFunctionPointer getDowncallStubPointer(NativeEntryPointInfo nep) {
         FunctionPointerHolder holder = downcallStubs.get(nep);
         if (holder == null) {
             throw reportMissingDowncall(nep);
         }
         return holder.functionPointer;
     }
 
+    CFunctionPointer getDowncallStubInvokerPointer(MethodType methodType) {
+        FunctionPointerHolder holder = downcallStubInvokers.get(methodType);
+        if (holder == null) {
+            throw reportMissingDowncall(methodType);
+        }
+        return holder.functionPointer;
+    }
+
     CFunctionPointer getUpcallStubPointer(JavaEntryPointInfo jep) {
         FunctionPointerHolder holder = upcallStubs.get(jep);
         if (holder == null) {
@@ -307,6 +327,9 @@ void freeTrampoline(long addr) {
     private MissingForeignRegistrationError reportMissingDowncall(NativeEntryPointInfo nep) {
         LinkRequest currentLinkRequest = null;
         for (LinkRequest linkRequest : currentLinkRequests) {
+            if (!Thread.currentThread().equals(linkRequest.requester)) {
+                continue;
+            }
             NativeEntryPointInfo nativeEntryPointInfo = abiUtils.makeNativeEntrypoint(linkRequest.functionDescriptor, linkRequest.linkerOptions);
             if (nep.equals(nativeEntryPointInfo)) {
                 currentLinkRequest = linkRequest;
@@ -316,6 +339,21 @@ private MissingForeignRegistrationError reportMissingDowncall(NativeEntryPointIn
         throw MissingForeignRegistrationUtils.report(false, currentLinkRequest, nep.methodType());
     }
 
+    /**
+     * Similar to {@link #reportMissingDowncall(NativeEntryPointInfo)} but only matches the
+     * requested {@link MethodType}.
+     */
+    private MissingForeignRegistrationError reportMissingDowncall(MethodType methodType) {
+        LinkRequest currentLinkRequest = null;
+        for (LinkRequest linkRequest : currentLinkRequests) {
+            if (methodType.equals(linkRequest.functionDescriptor.toMethodType())) {
+                currentLinkRequest = linkRequest;
+                break;
+            }
+        }
+        throw MissingForeignRegistrationUtils.report(false, currentLinkRequest, methodType);
+    }
+
     /**
      * Similar to {@link #reportMissingDowncall} but for upcalls.
      */
@@ -352,10 +390,10 @@ private static MissingForeignRegistrationError report(boolean upcall, LinkReques
                                         "upcallStub"));
     }
 
-    record LinkRequest(boolean upcall, FunctionDescriptor functionDescriptor, LinkerOptions linkerOptions) implements AutoCloseable, JsonPrintable {
+    record LinkRequest(boolean upcall, FunctionDescriptor functionDescriptor, LinkerOptions linkerOptions, Thread requester) implements AutoCloseable, JsonPrintable {
 
         static LinkRequest create(boolean upcall, FunctionDescriptor functionDescriptor, LinkerOptions linkerOptions) {
-            LinkRequest linkRequest = new LinkRequest(upcall, functionDescriptor, linkerOptions);
+            LinkRequest linkRequest = new LinkRequest(upcall, functionDescriptor, linkerOptions, Thread.currentThread());
             ForeignFunctionsRuntime.singleton().currentLinkRequests.push(linkRequest);
             return linkRequest;
         }
@@ -399,9 +437,10 @@ public void printJson(JsonWriter writer) throws IOException {
     @Override
     public Object linkToNative(Object... args) throws Throwable {
         Target_jdk_internal_foreign_abi_NativeEntryPoint nep = (Target_jdk_internal_foreign_abi_NativeEntryPoint) args[args.length - 1];
-        StubPointer pointer = Word.pointer(nep.downcallStubAddress);
-        /* The nep argument will be dropped in the invoked function */
-        return pointer.invoke(args);
+        StubInvokerPointer invoker = (StubInvokerPointer) nep.downcallInvokerPointer;
+        CFunctionPointer stub = nep.downcallStubPointer;
+        /* The nep argument will be dropped in the invoked downcall stub */
+        return invoker.invoke(stub, args);
     }
 
     @Override
@@ -437,6 +476,12 @@ public void onScopeReachable(Object scopeObj, DisallowedObjectReporter reporter)
         }
     }
 
+    @AlwaysInline("method handle interpreter performance")
+    @Override
+    public MethodType getMethodTypeFromNativeEntryPoint(Object nativeEntryPoint) {
+        return ((Target_jdk_internal_foreign_abi_NativeEntryPoint) nativeEntryPoint).type();
+    }
+
     /**
      * Workaround for CapturableState.maskFromName(String) being interruptible.
      */
@@ -502,7 +547,8 @@ public boolean isSafeCallee(ResolvedJavaMethod method) {
     }
 }
 
-interface StubPointer extends CFunctionPointer {
+/** Invoke interface for {@code com.oracle.svm.hosted.foreign.DowncallStubInvoker}. */
+interface StubInvokerPointer extends CFunctionPointer {
     @InvokeJavaFunctionPointer
-    Object invoke(Object... args);
+    Object invoke(CFunctionPointer downcallStub, Object... args);
 }