8281866: Enhance MethodHandle invocations

Reviewed-by: yan
Backport-of: d974d9da365f787f67971d88c79371c8b0769f75

hotspot/src/share/vm/interpreter/linkResolver.cpp

@@ -1209,22 +1209,41 @@ void LinkResolver::resolve_invokehandle(CallInfo& result, constantPoolHandle poo
     ResourceMark rm(THREAD);
     tty->print_cr("resolve_invokehandle %s %s", method_name->as_C_string(), method_signature->as_C_string());
   }
-  resolve_handle_call(result, resolved_klass, method_name, method_signature, current_klass, CHECK);
+  resolve_handle_call(result, resolved_klass, method_name, method_signature, current_klass, true, CHECK);
 }
 
 void LinkResolver::resolve_handle_call(CallInfo& result, KlassHandle resolved_klass,
                                        Symbol* method_name, Symbol* method_signature,
-                                       KlassHandle current_klass,
+                                       KlassHandle current_klass, bool check_access,
                                        TRAPS) {
   // JSR 292:  this must be an implicitly generated method MethodHandle.invokeExact(*...) or similar
   assert(resolved_klass() == SystemDictionary::MethodHandle_klass(), "");
   assert(MethodHandles::is_signature_polymorphic_name(method_name), "");
   methodHandle resolved_method;
-  Handle       resolved_appendix;
+  Handle resolved_appendix;
-  Handle       resolved_method_type;
+  Handle resolved_method_type;
   lookup_polymorphic_method(resolved_method, resolved_klass,
                             method_name, method_signature,
                             current_klass, &resolved_appendix, &resolved_method_type, CHECK);
+  if (check_access) {
+    vmIntrinsics::ID iid = MethodHandles::signature_polymorphic_name_id(method_name);
+    if (MethodHandles::is_signature_polymorphic_intrinsic(iid)) {
+      // Check if method can be accessed by the referring class.
+      // MH.linkTo* invocations are not rewritten to invokehandle.
+      assert(iid == vmIntrinsics::_invokeBasic, err_msg("%s", vmIntrinsics::name_at(iid)));
+
+      assert(current_klass.not_null(), "current_klass should not be null");
+      check_method_accessability(current_klass,
+                                 resolved_klass,
+                                 resolved_method->method_holder(),
+                                 resolved_method,
+                                 CHECK);
+    } else {
+      // Java code is free to arbitrarily link signature-polymorphic invokers.
+      assert(iid == vmIntrinsics::_invokeGeneric, err_msg("not an invoker: %s", vmIntrinsics::name_at(iid)));
+      assert(MethodHandles::is_signature_polymorphic_public_name(resolved_klass(), method_name), "not public");
+    }
+  }
   result.set_handle(resolved_method, resolved_appendix, resolved_method_type, CHECK);
 }
 

hotspot/src/share/vm/interpreter/linkResolver.hpp

@@ -162,7 +162,7 @@ class LinkResolver: AllStatic {
   static void resolve_special_call  (CallInfo& result,              KlassHandle resolved_klass, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, bool check_access, TRAPS);
   static void resolve_virtual_call  (CallInfo& result, Handle recv, KlassHandle recv_klass, KlassHandle resolved_klass, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, bool check_access, bool check_null_and_abstract, TRAPS);
   static void resolve_interface_call(CallInfo& result, Handle recv, KlassHandle recv_klass, KlassHandle resolved_klass, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, bool check_access, bool check_null_and_abstract, TRAPS);
-  static void resolve_handle_call   (CallInfo& result,                                      KlassHandle resolved_klass, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, TRAPS);
+  static void resolve_handle_call   (CallInfo& result,                                      KlassHandle resolved_klass, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, bool check_access, TRAPS);
   static void resolve_dynamic_call  (CallInfo& result,                                      Handle bootstrap_specifier, Symbol* method_name, Symbol* method_signature, KlassHandle current_klass, TRAPS);
 
   // same as above for compile-time resolution; but returns null handle instead of throwing an exception on error

hotspot/src/share/vm/prims/methodHandles.cpp

@@ -413,6 +413,24 @@ vmIntrinsics::ID MethodHandles::signature_polymorphic_name_id(klassOop klass, Sy
   return vmIntrinsics::_none;
 }
 
+// Returns true if method is signature polymorphic and public
+bool MethodHandles::is_signature_polymorphic_public_name(klassOop klass, Symbol* name) {
+  if (is_signature_polymorphic_name(klass, name)) {
+    instanceKlass* iklass = instanceKlass::cast(klass);
+    int me;
+    int ms = iklass->find_method_by_name(name, &me);
+    assert(ms != -1, "");
+    for (; ms < me; ms++) {
+      methodOop m = methodOop(iklass->methods()->obj_at(ms));
+      int required = JVM_ACC_NATIVE | JVM_ACC_VARARGS | JVM_ACC_PUBLIC;
+      int flags = m->access_flags().as_int();
+      if ((flags & required) == required && ArgumentCount(m->signature()).size() == 1) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
 
 // convert the external string or reflective type to an internal signature
 Symbol* MethodHandles::lookup_signature(oop type_str, bool intern_if_not_found, TRAPS) {
@@ -697,7 +715,7 @@ Handle MethodHandles::resolve_MemberName(Handle mname, KlassHandle caller, TRAPS
         } else if (mh_invoke_id != vmIntrinsics::_none) {
           assert(!is_signature_polymorphic_static(mh_invoke_id), "");
           LinkResolver::resolve_handle_call(result,
-                        defc, name, type, caller, THREAD);
+                        defc, name, type, caller, caller.not_null(), THREAD);
         } else if (ref_kind == JVM_REF_invokeSpecial) {
           do_dispatch = false;  // force non-virtual linkage
           LinkResolver::resolve_special_call(result,

hotspot/src/share/vm/prims/methodHandles.hpp

@@ -129,6 +129,7 @@ class MethodHandles: AllStatic {
   static bool is_signature_polymorphic_name(klassOop klass, Symbol* name) {
     return signature_polymorphic_name_id(klass, name) != vmIntrinsics::_none;
   }
+  static bool is_signature_polymorphic_public_name(klassOop klass, Symbol* name);
 
   enum {
     // format of query to getConstant: