Skip to content

Commit c5d10ef

Browse files
committed
Refactoring write barrier code
1 parent 414aca2 commit c5d10ef

File tree

4 files changed

+205
-92
lines changed

4 files changed

+205
-92
lines changed

src/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,9 @@ UV_HEADERS += uv/*.h
123123
endif
124124
PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,work-stealing-queue.h gc-interface.h gc-tls-common.h julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h jloptions.h)
125125
ifneq (${MMTK_PLAN},None)
126-
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h)
126+
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h gc-wb-mmtk.h)
127127
else
128-
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h)
128+
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h gc-wb-stock.h)
129129
endif
130130
ifeq ($(OS),WINNT)
131131
PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,win32_ucontext.h)

src/gc-wb-mmtk.h

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// This file is a part of Julia. License is MIT: https://julialang.org/license
2+
3+
/*
4+
write barriers which should be inlined by the compiler
5+
*/
6+
7+
#ifndef JL_GC_WB_H
8+
#define JL_GC_WB_H
9+
10+
#ifdef __cplusplus
11+
extern "C" {
12+
#endif
13+
14+
extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr);
15+
extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr);
16+
extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
17+
18+
#define MMTK_OBJECT_BARRIER (1)
19+
// Stickyimmix needs write barrier. Immix does not need write barrier.
20+
#ifdef MMTK_PLAN_IMMIX
21+
#define MMTK_NEEDS_WRITE_BARRIER (0)
22+
#endif
23+
#ifdef MMTK_PLAN_STICKYIMMIX
24+
#define MMTK_NEEDS_WRITE_BARRIER (1)
25+
#endif
26+
27+
// GC write barriers
28+
29+
// Directly call into MMTk for write barrier (debugging only)
30+
STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT
31+
{
32+
jl_task_t *ct = jl_current_task;
33+
jl_ptls_t ptls = ct->ptls;
34+
mmtk_object_reference_write_post(&ptls->gc_tls.mmtk_mutator, parent, ptr);
35+
}
36+
37+
// Inlined fastpath
38+
STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSAFEPOINT
39+
{
40+
if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
41+
intptr_t addr = (intptr_t) (void*) parent;
42+
uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6);
43+
intptr_t shift = (addr >> 3) & 0b111;
44+
uint8_t byte_val = *meta_addr;
45+
if (((byte_val >> shift) & 1) == 1) {
46+
jl_task_t *ct = jl_current_task;
47+
jl_ptls_t ptls = ct->ptls;
48+
mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, ptr);
49+
}
50+
}
51+
}
52+
53+
STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
54+
{
55+
mmtk_gc_wb_fast(parent, ptr);
56+
}
57+
58+
STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
59+
{
60+
mmtk_gc_wb_fast(ptr, (void*)0);
61+
}
62+
63+
STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
64+
{
65+
mmtk_gc_wb_fast(parent, (void*)0);
66+
}
67+
68+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
69+
jl_genericmemory_t *src, _Atomic(void*) * src_p,
70+
size_t* n) JL_NOTSAFEPOINT
71+
{
72+
mmtk_gc_wb_fast(dest_owner, (void*)0);
73+
}
74+
75+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
76+
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
77+
{
78+
mmtk_gc_wb_fast(owner, (void*)0);
79+
}
80+
81+
82+
#ifdef __cplusplus
83+
}
84+
#endif
85+
86+
#endif

src/gc-wb-stock.h

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// This file is a part of Julia. License is MIT: https://julialang.org/license
2+
3+
/*
4+
write barriers which should be inlined by the compiler
5+
*/
6+
7+
#ifndef JL_GC_WB_H
8+
#define JL_GC_WB_H
9+
10+
#ifdef __cplusplus
11+
extern "C" {
12+
#endif
13+
14+
// GC write barriers
15+
16+
STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
17+
{
18+
// parent and ptr isa jl_value_t*
19+
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
20+
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
21+
jl_gc_queue_root((jl_value_t*)parent);
22+
}
23+
24+
STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
25+
{
26+
// if ptr is old
27+
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
28+
jl_gc_queue_root((jl_value_t*)ptr);
29+
}
30+
}
31+
32+
STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
33+
{
34+
// 3 == GC_OLD_MARKED
35+
// ptr is an immutable object
36+
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
37+
return; // parent is young or in remset
38+
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
39+
return; // ptr is old and not in remset (thus it does not point to young)
40+
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
41+
const jl_datatype_layout_t *ly = dt->layout;
42+
if (ly->npointers)
43+
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
44+
}
45+
46+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
47+
jl_genericmemory_t *src, _Atomic(void*) * src_p,
48+
size_t* n) JL_NOTSAFEPOINT
49+
{
50+
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
51+
jl_value_t *src_owner = jl_genericmemory_owner(src);
52+
size_t done = 0;
53+
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
54+
if (dest_p < src_p || dest_p > src_p + (*n)) {
55+
for (; done < (*n); done++) { // copy forwards
56+
void *val = jl_atomic_load_relaxed(src_p + done);
57+
jl_atomic_store_release(dest_p + done, val);
58+
// `val` is young or old-unmarked
59+
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
60+
jl_gc_queue_root(dest_owner);
61+
break;
62+
}
63+
}
64+
src_p += done;
65+
dest_p += done;
66+
}
67+
else {
68+
for (; done < (*n); done++) { // copy backwards
69+
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
70+
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
71+
// `val` is young or old-unmarked
72+
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
73+
jl_gc_queue_root(dest_owner);
74+
break;
75+
}
76+
}
77+
}
78+
(*n) -= done;
79+
}
80+
}
81+
}
82+
83+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
84+
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
85+
{
86+
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
87+
jl_value_t *src_owner = jl_genericmemory_owner(src);
88+
size_t elsz = dt->layout->size;
89+
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
90+
dt = (jl_datatype_t*)jl_tparam1(dt);
91+
for (size_t done = 0; done < n; done++) { // copy forwards
92+
char* s = (char*)src_p+done*elsz;
93+
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
94+
jl_gc_queue_multiroot(owner, s, dt);
95+
}
96+
}
97+
}
98+
}
99+
100+
#ifdef __cplusplus
101+
}
102+
#endif
103+
104+
#endif

src/julia.h

Lines changed: 13 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666

6767
typedef struct _jl_taggedvalue_t jl_taggedvalue_t;
6868
typedef struct _jl_tls_states_t *jl_ptls_t;
69+
typedef struct _jl_genericmemory_t jl_genericmemory_t;
6970

7071
#ifdef JL_LIBRARY_EXPORTS
7172
#include "uv.h"
@@ -1244,6 +1245,18 @@ STATIC_INLINE jl_value_t *jl_svecset(
12441245
#define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length)
12451246
#define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a))
12461247

1248+
JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
1249+
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))
1250+
1251+
STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
1252+
1253+
// write barriers
1254+
#ifndef MMTK_GC
1255+
#include "gc-wb-stock.h"
1256+
#else
1257+
#include "gc-wb-mmtk.h"
1258+
#endif
1259+
12471260
/*
12481261
how - allocation style
12491262
0 = data is inlined
@@ -1300,94 +1313,6 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set(
13001313
}
13011314
#endif
13021315

1303-
// GC write barriers
1304-
1305-
STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
1306-
{
1307-
// parent and ptr isa jl_value_t*
1308-
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
1309-
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
1310-
jl_gc_queue_root((jl_value_t*)parent);
1311-
}
1312-
1313-
STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
1314-
{
1315-
// if ptr is old
1316-
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
1317-
jl_gc_queue_root((jl_value_t*)ptr);
1318-
}
1319-
}
1320-
1321-
STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
1322-
{
1323-
// 3 == GC_OLD_MARKED
1324-
// ptr is an immutable object
1325-
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
1326-
return; // parent is young or in remset
1327-
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
1328-
return; // ptr is old and not in remset (thus it does not point to young)
1329-
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
1330-
const jl_datatype_layout_t *ly = dt->layout;
1331-
if (ly->npointers)
1332-
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
1333-
}
1334-
1335-
STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
1336-
1337-
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
1338-
jl_genericmemory_t *src, _Atomic(void*) * src_p,
1339-
size_t* n) JL_NOTSAFEPOINT
1340-
{
1341-
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
1342-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1343-
size_t done = 0;
1344-
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
1345-
if (dest_p < src_p || dest_p > src_p + (*n)) {
1346-
for (; done < (*n); done++) { // copy forwards
1347-
void *val = jl_atomic_load_relaxed(src_p + done);
1348-
jl_atomic_store_release(dest_p + done, val);
1349-
// `val` is young or old-unmarked
1350-
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
1351-
jl_gc_queue_root(dest_owner);
1352-
break;
1353-
}
1354-
}
1355-
src_p += done;
1356-
dest_p += done;
1357-
}
1358-
else {
1359-
for (; done < (*n); done++) { // copy backwards
1360-
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
1361-
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
1362-
// `val` is young or old-unmarked
1363-
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
1364-
jl_gc_queue_root(dest_owner);
1365-
break;
1366-
}
1367-
}
1368-
}
1369-
(*n) -= done;
1370-
}
1371-
}
1372-
}
1373-
1374-
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
1375-
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
1376-
{
1377-
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
1378-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1379-
size_t elsz = dt->layout->size;
1380-
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
1381-
dt = (jl_datatype_t*)jl_tparam1(dt);
1382-
for (size_t done = 0; done < n; done++) { // copy forwards
1383-
char* s = (char*)src_p+done*elsz;
1384-
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
1385-
jl_gc_queue_multiroot(owner, s, dt);
1386-
}
1387-
}
1388-
}
1389-
}
1390-
13911316
STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT
13921317
{
13931318
jl_genericmemory_t *m_ = (jl_genericmemory_t*)m;
@@ -2402,8 +2327,6 @@ JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e JL_MAYBE_UNROOTED);
24022327
JL_DLLEXPORT void JL_NORETURN jl_rethrow(void);
24032328
JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED);
24042329
JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e, jl_task_t *ct);
2405-
JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
2406-
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))
24072330

24082331
extern JL_DLLIMPORT int jl_task_gcstack_offset;
24092332
extern JL_DLLIMPORT int jl_task_ptls_offset;

0 commit comments

Comments
 (0)