1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
#define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
#include "object.h"
#include "base/atomic.h"
#include "heap_poisoning.h"
#include "lock_word-inl.h"
#include "object_reference-inl.h"
#include "read_barrier.h"
#include "runtime.h"
namespace art HIDDEN {
namespace mirror {
template<VerifyObjectFlags kVerifyFlags>
inline LockWord Object::GetLockWord(bool as_volatile) {
if (as_volatile) {
return LockWord(GetField32Volatile<kVerifyFlags>(MonitorOffset()));
}
return LockWord(GetField32<kVerifyFlags>(MonitorOffset()));
}
template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
inline bool Object::CasField32(MemberOffset field_offset,
int32_t old_value,
int32_t new_value,
CASMode mode,
std::memory_order memory_order) {
if (kCheckTransaction) {
DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
}
if (kTransactionActive) {
Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
}
if (kVerifyFlags & kVerifyThis) {
VerifyObject(this);
}
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
return atomic_addr->CompareAndSet(old_value, new_value, mode, memory_order);
}
inline bool Object::CasLockWord(LockWord old_val,
LockWord new_val,
CASMode mode,
std::memory_order memory_order) {
// Force use of non-transactional mode and do not check.
return CasField32<false, false>(MonitorOffset(),
old_val.GetValue(),
new_val.GetValue(),
mode,
memory_order);
}
inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
if (!kUseBakerReadBarrier) {
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
#if defined(__arm__)
uintptr_t obj = reinterpret_cast<uintptr_t>(this);
uintptr_t result;
DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
// Use inline assembly to prevent the compiler from optimizing away the false dependency.
__asm__ __volatile__(
"ldr %[result], [%[obj], #4]\n\t"
// This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
// null, without them being able to assume that fact.
"eor %[fad], %[result], %[result]\n\t"
: [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
: [obj] "r" (obj));
DCHECK_EQ(*fake_address_dependency, 0U);
LockWord lw(static_cast<uint32_t>(result));
uint32_t rb_state = lw.ReadBarrierState();
return rb_state;
#elif defined(__aarch64__)
uintptr_t obj = reinterpret_cast<uintptr_t>(this);
uintptr_t result;
DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
// Use inline assembly to prevent the compiler from optimizing away the false dependency.
__asm__ __volatile__(
"ldr %w[result], [%[obj], #4]\n\t"
// This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
// null, without them being able to assume that fact.
"eor %[fad], %[result], %[result]\n\t"
: [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
: [obj] "r" (obj));
DCHECK_EQ(*fake_address_dependency, 0U);
LockWord lw(static_cast<uint32_t>(result));
uint32_t rb_state = lw.ReadBarrierState();
return rb_state;
#elif defined(__i386__) || defined(__x86_64__) || defined(__riscv)
// TODO(riscv64): add arch-specific implementation
LockWord lw = GetLockWord(false);
// i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
// reordering.
*fake_address_dependency = 0;
std::atomic_signal_fence(std::memory_order_acquire);
uint32_t rb_state = lw.ReadBarrierState();
return rb_state;
#else
UNUSED(fake_address_dependency);
LOG(FATAL) << "Unsupported architecture.";
UNREACHABLE();
#endif
}
inline uint32_t Object::GetReadBarrierState() {
if (!kUseBakerReadBarrier) {
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
DCHECK(kUseBakerReadBarrier);
LockWord lw(GetFieldPrimitive<uint32_t, /*kIsVolatile=*/false>(MonitorOffset()));
uint32_t rb_state = lw.ReadBarrierState();
DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
return rb_state;
}
inline uint32_t Object::GetReadBarrierStateAcquire() {
if (!kUseBakerReadBarrier) {
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
LockWord lw(GetFieldAcquire<uint32_t>(MonitorOffset()));
uint32_t rb_state = lw.ReadBarrierState();
DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
return rb_state;
}
inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state,
uint32_t rb_state,
std::memory_order order) {
if (!kUseBakerReadBarrier) {
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
LockWord expected_lw;
LockWord new_lw;
do {
LockWord lw = GetLockWord(false);
if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
// Lost the race.
return false;
}
expected_lw = lw;
expected_lw.SetReadBarrierState(expected_rb_state);
new_lw = lw;
new_lw.SetReadBarrierState(rb_state);
// ConcurrentCopying::ProcessMarkStackRef uses this with
// `kMemoryOrder` == `std::memory_order_release`.
// If `kMemoryOrder` == `std::memory_order_release`, use a CAS release so that when GC updates
// all the fields of an object and then changes the object from gray to black (non-gray), the
// field updates (stores) will be visible (won't be reordered after this CAS.)
} while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, order));
return true;
}
inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
LockWord expected_lw;
LockWord new_lw;
do {
LockWord lw = GetLockWord(false);
if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
// Lost the race.
return false;
}
expected_lw = lw;
new_lw = lw;
new_lw.SetMarkBitState(mark_bit);
// Since this is only set from the mutator, we can use the non-release CAS.
} while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, std::memory_order_relaxed));
return true;
}
} // namespace mirror
} // namespace art
#endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
|
