Skip to content

Commit fb2563d

Browse files
clairechingchingclairechingching
authored
[BPF] add allows-misaligned-mem-access target feature (llvm#167013)
This proposal adds a `cl::opt` CLI flag `-bpf-allow-misaligned-mem-access` to BPF target that lets users enable allowing misaligned memory accesses. The motivation behind the proposal is user space eBPF VMs (interpreters or JITs running in user space) typically run on real CPUs where unaligned memory accesses are acceptable (or handled efficiently) and can be enabled to simplify lowering and improve performance. In contrast, kernel eBPF must obey verifier constraints and platform-specific alignment restrictions. A new CLI option keeps kernel behavior unchanged while giving userspace VMs an explicit opt-in to enable more permissive codegen. It supports both use-cases without diverging codebases.
1 parent e63a47d commit fb2563d

File tree

6 files changed

+234
-0
lines changed

6 files changed

+234
-0
lines changed

‎llvm/lib/Target/BPF/BPF.td‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ def ALU32 : SubtargetFeature<"alu32", "HasAlu32", "true",
2727
def DwarfRIS: SubtargetFeature<"dwarfris", "UseDwarfRIS", "true",
2828
"Disable MCAsmInfo DwarfUsesRelocationsAcrossSections">;
2929

30+
def MisalignedMemAccess : SubtargetFeature<"allows-misaligned-mem-access",
31+
"AllowsMisalignedMemAccess", "true",
32+
"Allows misaligned memory access">;
33+
3034
def : Proc<"generic", []>;
3135
def : Proc<"v1", []>;
3236
def : Proc<"v2", []>;

‎llvm/lib/Target/BPF/BPFISelLowering.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,26 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
206206
HasJmp32 = STI.getHasJmp32();
207207
HasJmpExt = STI.getHasJmpExt();
208208
HasMovsx = STI.hasMovsx();
209+
210+
AllowsMisalignedMemAccess = STI.getAllowsMisalignedMemAccess();
211+
}
212+
213+
bool BPFTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
214+
MachineMemOperand::Flags,
215+
unsigned *Fast) const {
216+
// allows-misaligned-mem-access is disabled
217+
if (!AllowsMisalignedMemAccess)
218+
return false;
219+
220+
// only allow misalignment for simple value types
221+
if (!VT.isSimple())
222+
return false;
223+
224+
// always assume fast mode when misalignment is allowed
225+
if (Fast)
226+
*Fast = true;
227+
228+
return true;
209229
}
210230

211231
bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

‎llvm/lib/Target/BPF/BPFISelLowering.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ class BPFTargetLowering : public TargetLowering {
3232
// with the given GlobalAddress is legal.
3333
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
3434

35+
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
36+
MachineMemOperand::Flags,
37+
unsigned *) const override;
38+
3539
BPFTargetLowering::ConstraintType
3640
getConstraintType(StringRef Constraint) const override;
3741

@@ -61,6 +65,9 @@ class BPFTargetLowering : public TargetLowering {
6165
bool HasJmpExt;
6266
bool HasMovsx;
6367

68+
// Allows Misalignment
69+
bool AllowsMisalignedMemAccess;
70+
6471
SDValue LowerSDIVSREM(SDValue Op, SelectionDAG &DAG) const;
6572
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
6673
SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;

‎llvm/lib/Target/BPF/BPFSubtarget.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ void BPFSubtarget::initializeEnvironment() {
6969
HasStoreImm = false;
7070
HasLoadAcqStoreRel = false;
7171
HasGotox = false;
72+
AllowsMisalignedMemAccess = false;
7273
}
7374

7475
void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {

‎llvm/lib/Target/BPF/BPFSubtarget.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
6363
// whether we should enable MCAsmInfo DwarfUsesRelocationsAcrossSections
6464
bool UseDwarfRIS;
6565

66+
// whether we allows misaligned memory access
67+
bool AllowsMisalignedMemAccess;
68+
6669
// whether cpu v4 insns are enabled.
6770
bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm,
6871
HasLoadAcqStoreRel, HasGotox;
@@ -87,6 +90,9 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
8790
bool getHasJmp32() const { return HasJmp32; }
8891
bool getHasAlu32() const { return HasAlu32; }
8992
bool getUseDwarfRIS() const { return UseDwarfRIS; }
93+
bool getAllowsMisalignedMemAccess() const {
94+
return AllowsMisalignedMemAccess;
95+
}
9096
bool hasLdsx() const { return HasLdsx; }
9197
bool hasMovsx() const { return HasMovsx; }
9298
bool hasBswap() const { return HasBswap; }

‎llvm/test/CodeGen/BPF/unaligned_load_store.ll‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2+
3+
; RUN: llc -mtriple=bpfel -mattr=+allows-misaligned-mem-access -verify-machineinstrs %s -o - \
4+
; RUN: | FileCheck --check-prefixes=ALL,MISALIGN %s
5+
; RUN: llc -mtriple=bpfeb -mattr=+allows-misaligned-mem-access -verify-machineinstrs %s -o - \
6+
; RUN: | FileCheck --check-prefixes=ALL,MISALIGN %s
7+
8+
; RUN: llc -mtriple=bpfel -verify-machineinstrs %s -o - \
9+
; RUN: | FileCheck --check-prefixes=ALL,ALIGN %s
10+
; RUN: llc -mtriple=bpfeb -verify-machineinstrs %s -o - \
11+
; RUN: | FileCheck --check-prefixes=ALL,ALIGN %s
12+
; NOTE:
13+
; This test verifies that the new +bpf-allow-misaligned-mem-access
14+
; feature allows the BPF backend to emit direct unaligned load/store
15+
; instructions instead of byte-by-byte emulation sequences.
16+
17+
; ---------------------------------------------------------------------
18+
; i8 load
19+
; ---------------------------------------------------------------------
20+
define i8 @test_load_i8(i8* %p) {
21+
; ALL-LABEL: test_load_i8:
22+
; ALL: # %bb.0:
23+
; ALL-NEXT: w{{[0-9]+}} = *(u8 *)(r1 + 0)
24+
; ALL-NEXT: exit
25+
%v = load i8, i8* %p, align 1
26+
ret i8 %v
27+
}
28+
29+
; ---------------------------------------------------------------------
30+
; i8 store
31+
; ---------------------------------------------------------------------
32+
define void @test_store_i8(i8* %p, i8 %v) {
33+
; ALL-LABEL: test_store_i8:
34+
; ALL: # %bb.0:
35+
; ALL-NEXT: *(u8 *)(r1 + 0) = w{{[0-9]+}}
36+
; ALL-NEXT: exit
37+
store i8 %v, i8* %p, align 1
38+
ret void
39+
}
40+
41+
; ---------------------------------------------------------------------
42+
; i16 load
43+
; ---------------------------------------------------------------------
44+
define i16 @test_load_i16(i16* %p) {
45+
; MISALIGN-LABEL: test_load_i16:
46+
; MISALIGN: # %bb.0:
47+
; MISALIGN: w{{[0-9]+}} = *(u16 *)(r1 + 0)
48+
; MISALIGN: exit
49+
;
50+
; ALIGN-LABEL: test_load_i16:
51+
; ALIGN: # %bb.0:
52+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 0)
53+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 1)
54+
; ALIGN-DAG: w{{[0-9]+}} <<= 8
55+
; ALIGN-DAG: w{{[0-9]+}} |= w{{[0-9]+}}
56+
; ALIGN: exit
57+
%v = load i16, i16* %p, align 1
58+
ret i16 %v
59+
}
60+
61+
; ---------------------------------------------------------------------
62+
; i16 store
63+
; ---------------------------------------------------------------------
64+
define void @test_store_i16(i16* %p, i16 %v) {
65+
; MISALIGN-LABEL: test_store_i16:
66+
; MISALIGN: # %bb.0:
67+
; MISALIGN: *(u16 *)(r1 + 0) = w{{[0-9]+}}
68+
; MISALIGN: exit
69+
;
70+
; ALIGN-LABEL: test_store_i16:
71+
; ALIGN: # %bb.0:
72+
; ALIGN-DAG: *(u8 *)(r1 + 0) = w{{[0-9]+}}
73+
; ALIGN-DAG: w{{[0-9]+}} >>= 8
74+
; ALIGN-DAG: *(u8 *)(r1 + 1) = w{{[0-9]+}}
75+
; ALIGN: exit
76+
store i16 %v, i16* %p, align 1
77+
ret void
78+
}
79+
80+
; ---------------------------------------------------------------------
81+
; i32 load
82+
; ---------------------------------------------------------------------
83+
84+
define i32 @test_load_i32(i32* %p) {
85+
; MISALIGN-LABEL: test_load_i32:
86+
; MISALIGN: # %bb.0:
87+
; MISALIGN: w{{[0-9]+}} = *(u32 *)(r1 + 0)
88+
; MISALIGN: exit
89+
;
90+
; ALIGN-LABEL: test_load_i32:
91+
; ALIGN: # %bb.0:
92+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 0)
93+
; ALIGN-DAG: w{{[0-9]+}} <<= 8
94+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 1)
95+
; ALIGN-DAG: w{{[0-9]+}} |= w{{[0-9]+}}
96+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 2)
97+
; ALIGN-DAG: w{{[0-9]+}} <<= 16
98+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 3)
99+
; ALIGN-DAG: w{{[0-9]+}} <<= 24
100+
; ALIGN: exit
101+
%v = load i32, i32* %p, align 1
102+
ret i32 %v
103+
}
104+
105+
; ---------------------------------------------------------------------
106+
; i32 store
107+
; ---------------------------------------------------------------------
108+
109+
define void @test_store_i32(i32* %p, i32 %v) {
110+
; MISALIGN-LABEL: test_store_i32:
111+
; MISALIGN: # %bb.0:
112+
; MISALIGN: *(u32 *)(r1 + 0) = w{{[0-9]+}}
113+
; MISALIGN: exit
114+
;
115+
; ALIGN-LABEL: test_store_i32:
116+
; ALIGN: # %bb.0:
117+
; ALIGN-DAG: w{{[0-9]+}} = w{{[0-9]+}}
118+
; ALIGN-DAG: w{{[0-9]+}} >>= 24
119+
; ALIGN-DAG: *(u8 *)(r1 + 0) = w{{[0-9]+}}
120+
; ALIGN-DAG: w{{[0-9]+}} = w{{[0-9]+}}
121+
; ALIGN-DAG: w{{[0-9]+}} >>= 16
122+
; ALIGN-DAG: *(u8 *)(r1 + 1) = w{{[0-9]+}}
123+
; ALIGN-DAG: *(u8 *)(r1 + 2) = w{{[0-9]+}}
124+
; ALIGN-DAG: w{{[0-9]+}} >>= 8
125+
; ALIGN-DAG: *(u8 *)(r1 + 3) = w{{[0-9]+}}
126+
; ALIGN: exit
127+
store i32 %v, i32* %p, align 1
128+
ret void
129+
}
130+
131+
; ---------------------------------------------------------------------
132+
; i64 load
133+
; ---------------------------------------------------------------------
134+
135+
define i64 @test_load_i64(i64* %p) {
136+
; MISALIGN-LABEL: test_load_i64:
137+
; MISALIGN: # %bb.0:
138+
; MISALIGN: r0 = *(u64 *)(r1 + 0)
139+
; MISALIGN: exit
140+
;
141+
; ALIGN-LABEL: test_load_i64:
142+
; ALIGN: # %bb.0:
143+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 0)
144+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 1)
145+
; ALIGN-DAG: r{{[0-9]+}} <<= 8
146+
; ALIGN-DAG: r{{[0-9]+}} |= r{{[0-9]+}}
147+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 2)
148+
; ALIGN-DAG: r{{[0-9]+}} <<= 16
149+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 3)
150+
; ALIGN-DAG: r{{[0-9]+}} <<= 24
151+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 4)
152+
; ALIGN-DAG: w{{[0-9]+}} <<= 8
153+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 5)
154+
; ALIGN-DAG: w{{[0-9]+}} |= w{{[0-9]+}}
155+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 6)
156+
; ALIGN-DAG: w{{[0-9]+}} <<= 16
157+
; ALIGN-DAG: w{{[0-9]+}} = *(u8 *)(r1 + 7)
158+
; ALIGN-DAG: w{{[0-9]+}} <<= 24
159+
; ALIGN-DAG: r{{[0-9]+}} <<= 32
160+
; ALIGN: exit
161+
%v = load i64, i64* %p, align 1
162+
ret i64 %v
163+
}
164+
165+
; ---------------------------------------------------------------------
166+
; i64 store
167+
; ---------------------------------------------------------------------
168+
169+
define void @test_store_i64(i64* %p, i64 %v) {
170+
; MISALIGN-LABEL: test_store_i64:
171+
; MISALIGN: # %bb.0:
172+
; MISALIGN: *(u64 *)(r1 + 0) = r2
173+
; MISALIGN: exit
174+
;
175+
; ALIGN-LABEL: test_store_i64:
176+
; ALIGN: # %bb.0:
177+
; ALIGN-DAG: *(u8 *)(r1 + 0) = w{{[0-9]+}}
178+
; ALIGN-DAG: r{{[0-9]+}} = r{{[0-9]+}}
179+
; ALIGN-DAG: r{{[0-9]+}} >>= 56
180+
; ALIGN-DAG: *(u8 *)(r1 + 1) = w{{[0-9]+}}
181+
; ALIGN-DAG: r{{[0-9]+}} >>= 48
182+
; ALIGN-DAG: *(u8 *)(r1 + 2) = w{{[0-9]+}}
183+
; ALIGN-DAG: r{{[0-9]+}} >>= 40
184+
; ALIGN-DAG: *(u8 *)(r1 + 3) = w{{[0-9]+}}
185+
; ALIGN-DAG: r{{[0-9]+}} >>= 32
186+
; ALIGN-DAG: *(u8 *)(r1 + 4) = w{{[0-9]+}}
187+
; ALIGN-DAG: r{{[0-9]+}} >>= 24
188+
; ALIGN-DAG: *(u8 *)(r1 + 5) = w{{[0-9]+}}
189+
; ALIGN-DAG: r{{[0-9]+}} >>= 16
190+
; ALIGN-DAG: *(u8 *)(r1 + 6) = w{{[0-9]+}}
191+
; ALIGN-DAG: r{{[0-9]+}} >>= 8
192+
; ALIGN-DAG: *(u8 *)(r1 + 7) = w{{[0-9]+}}
193+
; ALIGN: exit
194+
store i64 %v, i64* %p, align 1
195+
ret void
196+
}

0 commit comments

Comments
 (0)