Skip to content

Commit a4ac434

Browse files
ilinpvilinpv
committed
[AArch64] Compiler-rt interface for out-of-line atomics.
Out-of-line helper functions to support LSE deployment added. This is a port of libgcc implementation: https://gcc.gnu.org/git/?p=gcc.git;h=33befddcb849235353dc263db1c7d07dc15c9faa Differential Revision: https://reviews.llvm.org/D91156
1 parent 2c63e76 commit a4ac434

File tree

5 files changed

+375
-16
lines changed

5 files changed

+375
-16
lines changed

‎compiler-rt/cmake/builtin-config-ix.cmake

+6
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ int foo(int x, int y) {
2323
")
2424

2525

26+
builtin_check_c_compiler_source(COMPILER_RT_HAS_ASM_LSE
27+
"
28+
asm(\".arch armv8-a+lse\");
29+
asm(\"cas w0, w1, [x2]\");
30+
")
31+
2632
set(ARM64 aarch64)
2733
set(ARM32 arm armhf armv6m armv7m armv7em armv7 armv7s armv7k)
2834
set(HEXAGON hexagon)

‎compiler-rt/lib/builtins/CMakeLists.txt

+30
Original file line numberDiff line numberDiff line change
@@ -502,9 +502,39 @@ endif()
502502
set(aarch64_SOURCES
503503
${GENERIC_TF_SOURCES}
504504
${GENERIC_SOURCES}
505+
cpu_model.c
505506
aarch64/fp_mode.c
506507
)
507508

509+
# Generate outline atomics helpers from lse.S base
510+
set(CUSTOM_FLAGS ${CMAKE_C_FLAGS})
511+
if(NOT ANDROID)
512+
append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG -DVISIBILITY_HIDDEN CUSTOM_FLAGS)
513+
endif()
514+
append_list_if(COMPILER_RT_HAS_ASM_LSE -DHAS_ASM_LSE CUSTOM_FLAGS)
515+
string(REPLACE " " "\t" CUSTOM_FLAGS "${CUSTOM_FLAGS}")
516+
517+
foreach(pat cas swp ldadd ldclr ldeor ldset)
518+
foreach(size 1 2 4 8 16)
519+
foreach(model 1 2 3 4)
520+
if(pat STREQUAL "cas" OR NOT size STREQUAL "16")
521+
set(helper_asm outline_atomic_${pat}${size}_${model}.S)
522+
add_custom_command(
523+
OUTPUT ${helper_asm}
524+
COMMAND ${CMAKE_C_COMPILER} -E ${CUSTOM_FLAGS} -DL_${pat} -DSIZE=${size} -DMODEL=${model}
525+
${CMAKE_CURRENT_SOURCE_DIR}/aarch64/lse.S -o ${helper_asm}
526+
DEPENDS aarch64/lse.S assembly.h
527+
)
528+
set_source_files_properties(${helper_asm} PROPERTIES GENERATED TRUE)
529+
set(aarch64_SOURCES
530+
${aarch64_SOURCES}
531+
${helper_asm}
532+
)
533+
endif()
534+
endforeach(model)
535+
endforeach(size)
536+
endforeach(pat)
537+
508538
if (MINGW)
509539
set(aarch64_SOURCES
510540
${aarch64_SOURCES}

‎compiler-rt/lib/builtins/aarch64/lse.S

+227
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2+
// See https://llvm.org/LICENSE.txt for license information.
3+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4+
5+
#include "../assembly.h"
6+
7+
// Out-of-line LSE atomics helpers. Ported from libgcc library.
8+
// N = {1, 2, 4, 8}
9+
// M = {1, 2, 4, 8, 16}
10+
// ORDER = {'relax', 'acq', 'rel', 'acq_rel'}
11+
// Routines implemented:
12+
//
13+
// iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
14+
// iN __aarch64_swpN_ORDER(iN val, iN *ptr)
15+
// iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
16+
// iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
17+
// iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
18+
// iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)
19+
//
20+
// Routines may modify temporary registers tmp0, tmp1, tmp2,
21+
// return value x0 and the flags only.
22+
23+
#ifdef __aarch64__
24+
25+
#ifdef HAS_ASM_LSE
26+
.arch armv8-a+lse
27+
#else
28+
.arch armv8-a
29+
#endif
30+
31+
HIDDEN(__aarch64_have_lse_atomics)
32+
33+
// Generate mnemonics for
34+
// L_cas: SIZE: 1,2,4,8,16 MODEL: 1,2,3,4
35+
// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8 MODEL: 1,2,3,4
36+
37+
#if SIZE == 1
38+
#define S b
39+
#define UXT uxtb
40+
#define B 0x00000000
41+
#elif SIZE == 2
42+
#define S h
43+
#define UXT uxth
44+
#define B 0x40000000
45+
#elif SIZE == 4 || SIZE == 8 || SIZE == 16
46+
#define S
47+
#define UXT mov
48+
#if SIZE == 4
49+
#define B 0x80000000
50+
#elif SIZE == 8
51+
#define B 0xc0000000
52+
#endif
53+
#else
54+
#error
55+
#endif // SIZE
56+
57+
#if MODEL == 1
58+
#define SUFF _relax
59+
#define A
60+
#define L
61+
#define M 0x000000
62+
#define N 0x000000
63+
#elif MODEL == 2
64+
#define SUFF _acq
65+
#define A a
66+
#define L
67+
#define M 0x400000
68+
#define N 0x800000
69+
#elif MODEL == 3
70+
#define SUFF _rel
71+
#define A
72+
#define L l
73+
#define M 0x008000
74+
#define N 0x400000
75+
#elif MODEL == 4
76+
#define SUFF _acq_rel
77+
#define A a
78+
#define L l
79+
#define M 0x408000
80+
#define N 0xc00000
81+
#else
82+
#error
83+
#endif // MODEL
84+
85+
// Define register size.
86+
#define x(N) GLUE2(x, N)
87+
#define w(N) GLUE2(w, N)
88+
#if SIZE < 8
89+
#define s(N) w(N)
90+
#else
91+
#define s(N) x(N)
92+
#endif
93+
94+
#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)
95+
#define LDXR GLUE4(ld, A, xr, S)
96+
#define STXR GLUE4(st, L, xr, S)
97+
98+
// Define temporary registers.
99+
#define tmp0 16
100+
#define tmp1 17
101+
#define tmp2 15
102+
103+
// Macro for branch to label if no LSE available
104+
.macro JUMP_IF_NOT_LSE label
105+
adrp x(tmp0), __aarch64_have_lse_atomics
106+
ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]
107+
cbz w(tmp0), \label
108+
.endm
109+
110+
#ifdef L_cas
111+
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas))
112+
JUMP_IF_NOT_LSE 8f
113+
#if SIZE < 16
114+
#ifdef HAS_ASM_LSE
115+
#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2]
116+
#else
117+
#define CAS .inst 0x08a07c41 + B + M
118+
#endif
119+
CAS // s(0), s(1), [x2]
120+
ret
121+
8:
122+
UXT s(tmp0), s(0)
123+
0:
124+
LDXR s(0), [x2]
125+
cmp s(0), s(tmp0)
126+
bne 1f
127+
STXR w(tmp1), s(1), [x2]
128+
cbnz w(tmp1), 0b
129+
1:
130+
ret
131+
#else
132+
#define LDXP GLUE3(ld, A, xp)
133+
#define STXP GLUE3(st, L, xp)
134+
#ifdef HAS_ASM_LSE
135+
#define CASP GLUE3(casp, A, L) x0, x1, x2, x3, [x4]
136+
#else
137+
#define CASP .inst 0x48207c82 + M
138+
#endif
139+
140+
CASP // x0, x1, x2, x3, [x4]
141+
ret
142+
8:
143+
mov x(tmp0), x0
144+
mov x(tmp1), x1
145+
0:
146+
LDXP x0, x1, [x4]
147+
cmp x0, x(tmp0)
148+
ccmp x1, x(tmp1), #0, eq
149+
bne 1f
150+
STXP w(tmp2), x2, x3, [x4]
151+
cbnz w(tmp2), 0b
152+
1:
153+
ret
154+
#endif
155+
END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))
156+
#endif // L_cas
157+
158+
#ifdef L_swp
159+
#ifdef HAS_ASM_LSE
160+
#define SWP GLUE4(swp, A, L, S) s(0), s(0), [x1]
161+
#else
162+
#define SWP .inst 0x38208020 + B + N
163+
#endif
164+
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))
165+
JUMP_IF_NOT_LSE 8f
166+
SWP // s(0), s(0), [x1]
167+
ret
168+
8:
169+
mov s(tmp0), s(0)
170+
0:
171+
LDXR s(0), [x1]
172+
STXR w(tmp1), s(tmp0), [x1]
173+
cbnz w(tmp1), 0b
174+
ret
175+
END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))
176+
#endif // L_swp
177+
178+
#if defined(L_ldadd) || defined(L_ldclr) || \
179+
defined(L_ldeor) || defined(L_ldset)
180+
181+
#ifdef L_ldadd
182+
#define LDNM ldadd
183+
#define OP add
184+
#define OPN 0x0000
185+
#elif defined(L_ldclr)
186+
#define LDNM ldclr
187+
#define OP bic
188+
#define OPN 0x1000
189+
#elif defined(L_ldeor)
190+
#define LDNM ldeor
191+
#define OP eor
192+
#define OPN 0x2000
193+
#elif defined(L_ldset)
194+
#define LDNM ldset
195+
#define OP orr
196+
#define OPN 0x3000
197+
#else
198+
#error
199+
#endif
200+
201+
#ifdef HAS_ASM_LSE
202+
#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1]
203+
#else
204+
#define LDOP .inst 0x38200020 + OPN + B + N
205+
#endif
206+
207+
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))
208+
JUMP_IF_NOT_LSE 8f
209+
LDOP // s(0), s(0), [x1]
210+
ret
211+
8:
212+
mov s(tmp0), s(0)
213+
0:
214+
LDXR s(0), [x1]
215+
OP s(tmp1), s(0), s(tmp0)
216+
STXR w(tmp2), s(tmp1), [x1]
217+
cbnz w(tmp2), 0b
218+
ret
219+
END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
220+
#endif // L_ldadd L_ldclr L_ldeor L_ldset
221+
222+
NO_EXEC_STACK_DIRECTIVE
223+
224+
// GNU property note for BTI and PAC
225+
GNU_PROPERTY_BTI_PAC
226+
227+
#endif // __aarch64__

0 commit comments

Comments
 (0)