| // |
| // Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. |
| // Copyright (c) 2021 SAP SE. All rights reserved. |
| // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| // |
| // This code is free software; you can redistribute it and/or modify it |
| // under the terms of the GNU General Public License version 2 only, as |
| // published by the Free Software Foundation. |
| // |
| // This code is distributed in the hope that it will be useful, but WITHOUT |
| // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| // version 2 for more details (a copy is included in the LICENSE file that |
| // accompanied this code). |
| // |
| // You should have received a copy of the GNU General Public License version |
| // 2 along with this work; if not, write to the Free Software Foundation, |
| // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| // |
| // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| // or visit www.oracle.com if you need additional information or have any |
| // questions. |
| // |
| |
| source_hpp %{ |
| |
| #include "gc/shared/gc_globals.hpp" |
| #include "gc/x/c2/xBarrierSetC2.hpp" |
| #include "gc/x/xThreadLocalData.hpp" |
| |
| %} |
| |
| source %{ |
| |
| static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, |
| Register tmp, uint8_t barrier_data) { |
| if (barrier_data == XLoadBarrierElided) { |
| return; |
| } |
| |
| XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data); |
| __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread); |
| __ and_(tmp, tmp, ref); |
| __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate); |
| __ bind(*stub->continuation()); |
| } |
| |
| static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, |
| Register tmp) { |
| XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong); |
| __ b(*stub->entry()); |
| __ bind(*stub->continuation()); |
| } |
| |
| static void x_compare_and_swap(MacroAssembler& _masm, const MachNode* node, |
| Register res, Register mem, Register oldval, Register newval, |
| Register tmp_xchg, Register tmp_mask, |
| bool weak, bool acquire) { |
| // z-specific load barrier requires strong CAS operations. |
| // Weak CAS operations are thus only emitted if the barrier is elided. |
| __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem, |
| MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, nullptr, true, |
| weak && node->barrier_data() == XLoadBarrierElided); |
| |
| if (node->barrier_data() != XLoadBarrierElided) { |
| Label skip_barrier; |
| |
| __ ld(tmp_mask, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread); |
| __ and_(tmp_mask, tmp_mask, tmp_xchg); |
| __ beq(CCR0, skip_barrier); |
| |
| // CAS must have failed because pointer in memory is bad. |
| x_load_barrier_slow_path(_masm, node, Address(mem), tmp_xchg, res /* used as tmp */); |
| |
| __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem, |
| MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, nullptr, true, weak); |
| |
| __ bind(skip_barrier); |
| } |
| |
| if (acquire) { |
| if (support_IRIW_for_not_multiple_copy_atomic_cpu) { |
| // Uses the isync instruction as an acquire barrier. |
| // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). |
| __ isync(); |
| } else { |
| __ sync(); |
| } |
| } |
| } |
| |
| static void x_compare_and_exchange(MacroAssembler& _masm, const MachNode* node, |
| Register res, Register mem, Register oldval, Register newval, Register tmp, |
| bool weak, bool acquire) { |
| // z-specific load barrier requires strong CAS operations. |
| // Weak CAS operations are thus only emitted if the barrier is elided. |
| __ cmpxchgd(CCR0, res, oldval, newval, mem, |
| MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, nullptr, true, |
| weak && node->barrier_data() == XLoadBarrierElided); |
| |
| if (node->barrier_data() != XLoadBarrierElided) { |
| Label skip_barrier; |
| __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread); |
| __ and_(tmp, tmp, res); |
| __ beq(CCR0, skip_barrier); |
| |
| x_load_barrier_slow_path(_masm, node, Address(mem), res, tmp); |
| |
| __ cmpxchgd(CCR0, res, oldval, newval, mem, |
| MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, nullptr, true, weak); |
| |
| __ bind(skip_barrier); |
| } |
| |
| if (acquire) { |
| if (support_IRIW_for_not_multiple_copy_atomic_cpu) { |
| // Uses the isync instruction as an acquire barrier. |
| // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). |
| __ isync(); |
| } else { |
| __ sync(); |
| } |
| } |
| } |
| |
| %} |
| |
| instruct xLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0) |
| %{ |
| match(Set dst (LoadP mem)); |
| effect(TEMP_DEF dst, TEMP tmp, KILL cr0); |
| ins_cost(MEMORY_REF_COST); |
| |
| predicate((UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0) |
| && (n->as_Load()->is_unordered() || followed_by_acquire(n))); |
| |
| format %{ "LD $dst, $mem" %} |
| ins_encode %{ |
| assert($mem$$index == 0, "sanity"); |
| __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register); |
| x_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data()); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| // Load Pointer Volatile |
| instruct xLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0) |
| %{ |
| match(Set dst (LoadP mem)); |
| effect(TEMP_DEF dst, TEMP tmp, KILL cr0); |
| ins_cost(3 * MEMORY_REF_COST); |
| |
| // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation |
| predicate(UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0); |
| |
| format %{ "LD acq $dst, $mem" %} |
| ins_encode %{ |
| __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register); |
| x_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data()); |
| |
| // Uses the isync instruction as an acquire barrier. |
| // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). |
| __ isync(); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ |
| match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst)); |
| |
| format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %} |
| ins_encode %{ |
| x_compare_and_swap(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, |
| $tmp_xchg$$Register, $tmp_mask$$Register, |
| false /* weak */, false /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ |
| match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst)); |
| |
| format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %} |
| ins_encode %{ |
| x_compare_and_swap(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, |
| $tmp_xchg$$Register, $tmp_mask$$Register, |
| false /* weak */, true /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ |
| match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst); |
| |
| format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %} |
| ins_encode %{ |
| x_compare_and_swap(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, |
| $tmp_xchg$$Register, $tmp_mask$$Register, |
| true /* weak */, false /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ |
| match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst)); |
| |
| format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %} |
| ins_encode %{ |
| x_compare_and_swap(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, |
| $tmp_xchg$$Register, $tmp_mask$$Register, |
| true /* weak */, true /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp, flagsRegCR0 cr0) %{ |
| match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && ( |
| ((CompareAndSwapNode*)n)->order() != MemNode::acquire |
| && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst |
| )); |
| |
| format %{ "CMPXCHG $res, $mem, $oldval, $newval; as ptr; ptr" %} |
| ins_encode %{ |
| x_compare_and_exchange(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register, |
| false /* weak */, false /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, |
| iRegPdst tmp, flagsRegCR0 cr0) %{ |
| match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
| effect(TEMP_DEF res, TEMP tmp, KILL cr0); |
| |
| predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong) |
| && ( |
| ((CompareAndSwapNode*)n)->order() == MemNode::acquire |
| || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst |
| )); |
| |
| format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as ptr; ptr" %} |
| ins_encode %{ |
| x_compare_and_exchange(_masm, this, |
| $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register, |
| false /* weak */, true /* acquire */); |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
| |
| instruct xGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0) %{ |
| match(Set res (GetAndSetP mem newval)); |
| effect(TEMP_DEF res, TEMP tmp, KILL cr0); |
| |
| predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() != 0); |
| |
| format %{ "GetAndSetP $res, $mem, $newval" %} |
| ins_encode %{ |
| __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update()); |
| x_load_barrier(_masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data()); |
| |
| if (support_IRIW_for_not_multiple_copy_atomic_cpu) { |
| __ isync(); |
| } else { |
| __ sync(); |
| } |
| %} |
| ins_pipe(pipe_class_default); |
| %} |
