diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 41c736b0d11..c428420e480 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -701,6 +701,7 @@ void CallGenerator::do_late_inline_helper() { // field of the inline type. Build InlineTypeNodes from the inline type arguments. GraphKit arg_kit(jvms, &gvn); Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null()); + map = arg_kit.map(); map->set_control(arg_kit.control()); map->set_argument(jvms, i1, vt); } else { diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp index 8b1b2e13824..1fe99aff6be 100644 --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -695,7 +695,9 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (add_to_worklist) { igvn->add_users_to_worklist(this); // Check for further allowed opts } - for (DUIterator_Last imin, i = last_outs(imin); i >= imin; --i) { + uint nb_edges = 1; + for (DUIterator_Last imin, i = last_outs(imin); i >= imin; i -= nb_edges) { + nb_edges = 1; Node* n = last_out(i); igvn->hash_delete(n); // Remove from worklist before modifying edges if (n->outcnt() == 0) { @@ -2140,6 +2142,203 @@ bool PhiNode::wait_for_region_igvn(PhaseGVN* phase) { // Push inline type input nodes (and null) down through the phi recursively (can handle data loops). InlineTypeNode* PhiNode::push_inline_types_down(PhaseGVN* phase, bool can_reshape, ciInlineKlass* inline_klass) { + if (can_reshape) { + ResourceMark rm; + Unique_Node_List wq, wq2; + Node_List clones; + wq.push(this); + for (uint i = 0; i < wq.size(); ++i) { + Node* n = wq.at(i); + if (n->is_Phi()) { + for (uint j = 1; j < n->req(); ++j) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + wq.push(in); + } + } else if (n->is_ConstraintCast()) { + Node* in = n->in(1); + if (in == nullptr) { + continue; + } + wq.push(in); + } + } + for (int i = wq.size() - 1; i >= 0; i--) { + Node* n = wq.at(i); + if (!n->is_InlineType()) { + wq.remove(i); + } + } + uint init_nodes = wq.size(); + for (uint i = 0; i < wq.size(); ++i) { + Node* n = wq.at(i); + if (n->is_Phi()) { + for (uint j = 1; j < n->req(); ++j) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + wq.push(in); + } + } else if (n->is_ConstraintCast()) { + Node* in = n->in(1); + if (in == nullptr) { + continue; + } + wq.push(in); + } else if (n->is_InlineType()) { + Node* buf = n->as_InlineType()->get_oop(); + if (buf == nullptr) { + continue; + } + wq.push(buf); + wq2.push(n); + } + } + for (uint i = 0; i < wq2.size(); ++i) { + Node* n = wq2.at(i); + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* u = n->fast_out(i); + if (wq.member(u)) { + wq2.push(u); + } + } + } + for (uint i = 0; i < wq2.size(); ++i) { + Node* n = wq2.at(i); + assert(clones[n->_idx] == nullptr, ""); + if (n->is_InlineType()) { + clones.map(n->_idx, n->as_InlineType()->get_oop()); + } else { + Node* clone = n->clone(); + phase->is_IterGVN()->register_new_node_with_optimizer(clone); + clones.map(n->_idx, clone); + } + } + auto get_clone = [&](Node* n) { + Node* clone = nullptr; + while (true) { + Node* m = clones[n->_idx]; + if (m == nullptr) { + return clone; + } + clone = m; + n = clone; + } + }; + for (uint i = 0; i < wq2.size(); ++i) { + Node* n = wq2.at(i); + Node* n_clone = get_clone(n); + assert(n_clone != nullptr, ""); + if (n->is_Phi()) { + for (uint j = 1; j < n->req(); ++j) { + Node* in = n->in(j); + Node* in_clone = get_clone(in); + if (in_clone != nullptr) { + n_clone->set_req(j, in_clone); + } + } + } else if (n->is_ConstraintCast()) { + Node* in = n->in(1); + Node* in_clone = get_clone(in); + assert(in_clone != nullptr, ""); + n_clone->set_req(1, in_clone); + } else if (n->is_InlineType()) { + Node* in = n->as_InlineType()->get_oop(); + Node* in_clone = get_clone(in); + if (in_clone != nullptr) { + phase->is_IterGVN()->rehash_node_delayed(n); + n->as_InlineType()->set_oop(*phase, in_clone); + } + } + } +#ifdef ASSERT + { + uint vts_to_skip = 0; + uint before_phis = 0; + uint before_casts = 0; + for (uint i = 0; i < wq.size(); ++i) { + Node* n = wq.at(i); + if (n->is_Phi()) { + before_phis++; + } else if (n->is_ConstraintCast()) { + before_casts++; + } else if (n->is_InlineType()) { + Node* buf = n->as_InlineType()->get_oop(); + if (buf == nullptr) { + continue; + } + if (i >= init_nodes) { + vts_to_skip++; + } + } + } + + Unique_Node_List after; + after.push(this); + for (uint i = 0; i < after.size(); ++i) { + Node* n = after.at(i); + if (n->is_Phi()) { + for (uint j = 1; j < n->req(); ++j) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + after.push(in); + } + } else if (n->is_ConstraintCast()) { + Node* in = n->in(1); + if (in == nullptr) { + continue; + } + after.push(in); + } + } + for (int i = after.size() - 1; i >= 0; i--) { + Node* n = after.at(i); + if (!n->is_InlineType()) { + after.remove(i); + } + } + uint after_phis = 0; + uint after_casts = 0; + uint init_nodes = after.size(); + for (uint i = 0; i < after.size(); ++i) { + Node* n = after.at(i); + if (n->is_Phi()) { + after_phis++; + for (uint j = 1; j < n->req(); ++j) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + after.push(in); + } + } else if (n->is_ConstraintCast()) { + after_casts++; + Node* in = n->in(1); + if (in == nullptr) { + continue; + } + after.push(in); + } else if (n->is_InlineType()) { + assert(i < init_nodes, ""); + Node* buf = n->as_InlineType()->get_oop(); + if (buf == nullptr) { + continue; + } + after.push(buf); + } + } + assert(after.size() + vts_to_skip == wq.size(), ""); + assert(before_casts == after_casts, ""); + assert(before_phis == after_phis, ""); + } +#endif + } + assert(inline_klass != nullptr, "must be"); InlineTypeNode* vt = InlineTypeNode::make_null(*phase, inline_klass, /* transform = */ false)->clone_with_phis(phase, in(0), nullptr, !_type->maybe_null(), true); if (can_reshape) { @@ -2171,7 +2370,6 @@ InlineTypeNode* PhiNode::push_inline_types_down(PhaseGVN* phase, bool can_reshap } while (casts.size() != 0) { // Push the cast(s) through the InlineTypeNode - // TODO 8302217 Can we avoid cloning? See InlineTypeNode::clone_if_required Node* cast = casts.pop()->clone(); cast->set_req_X(1, n->as_InlineType()->get_oop(), phase); n = n->clone(); @@ -2951,21 +3149,6 @@ bool PhiNode::can_push_inline_types_down(PhaseGVN* phase, const bool can_reshape } inline_klass = nullptr; - // TODO 8302217 We need to prevent endless pushing through - bool only_phi = (outcnt() != 0); - for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { - Node* n = fast_out(i); - if (n->is_InlineType() && n->in(1) == this) { - return false; - } - if (!n->is_Phi()) { - only_phi = false; - } - } - if (only_phi) { - return false; - } - ResourceMark rm; Unique_Node_List worklist; worklist.push(this); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 7a57722c02a..abf7e2f9fa5 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2135,8 +2135,6 @@ void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) { // Verify that inline type is buffered when replacing by oop else if (u->is_InlineType()) { // InlineType uses don't need buffering because they are about to be replaced as well - } else if (u->is_Phi()) { - // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through } else { must_be_buffered = true; } diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 8f31e02278f..7998a8e0833 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -798,10 +798,6 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { for (uint j = 1; j < region->req(); j++) { Node *proj = region->in(j); Node *inp = phi->in(j); - if (inp->isa_InlineType()) { - // TODO 8302217 This prevents PhiNode::push_inline_types_through - return nullptr; - } if (get_ctrl(inp) == proj) { // Found local op cost++; // Check for a chain of dependent ops; these will all become diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java index d31eb43abee..5c90e7a792e 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java @@ -456,7 +456,6 @@ public void test8_verifier() { } // merge of inline types in a loop, stored in an object local - /* TODO 8302217: Enable again when this is fixed. @Test public Object test9() { Object o = valueField1; @@ -471,7 +470,6 @@ public Object test9() { public void test9_verifier() { Asserts.assertEQ(test9(), MyValue1.setX(valueField1, valueField1.x + 7)); } - */ // merge of inline types in an object local @ForceInline