Skip to content

Commit 66b18e5

Browse files
committedFeb 7, 2019
[BPF] add code-gen support for JMP32 instructions
JMP32 instructions has been added to eBPF ISA. They are 32-bit variants of existing BPF conditional jump instructions, but the comparison happens on low 32-bit sub-register only, therefore some unnecessary extensions could be saved. JMP32 instructions will only be available for -mcpu=v3. Host probe hook has been updated accordingly. JMP32 instructions will only be enabled in code-gen when -mattr=+alu32 enabled, meaning compiling the program using sub-register mode. For JMP32 encoding, it is a new instruction class, and is using the reserved eBPF class number 0x6. This patch has been tested by compiling and running kernel bpf selftests with JMP32 enabled. Acked-by: Yonghong Song <yhs@fb.com> Signed-off-by: Jiong Wang <jiong.wang@netronome.com> llvm-svn: 353384
1 parent 638110a commit 66b18e5

File tree

10 files changed

+179
-49
lines changed

10 files changed

+179
-49
lines changed
 

‎llvm/lib/Support/Host.cpp

+28-3
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,19 @@ StringRef sys::detail::getHostCPUNameForBPF() {
330330
#if !defined(__linux__) || !defined(__x86_64__)
331331
return "generic";
332332
#else
333-
uint8_t insns[40] __attribute__ ((aligned (8))) =
333+
uint8_t v3_insns[40] __attribute__ ((aligned (8))) =
334+
/* BPF_MOV64_IMM(BPF_REG_0, 0) */
335+
{ 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
336+
/* BPF_MOV64_IMM(BPF_REG_2, 1) */
337+
0xb7, 0x2, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
338+
/* BPF_JMP32_REG(BPF_JLT, BPF_REG_0, BPF_REG_2, 1) */
339+
0xae, 0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0,
340+
/* BPF_MOV64_IMM(BPF_REG_0, 1) */
341+
0xb7, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
342+
/* BPF_EXIT_INSN() */
343+
0x95, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
344+
345+
uint8_t v2_insns[40] __attribute__ ((aligned (8))) =
334346
/* BPF_MOV64_IMM(BPF_REG_0, 0) */
335347
{ 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
336348
/* BPF_MOV64_IMM(BPF_REG_2, 1) */
@@ -355,10 +367,23 @@ StringRef sys::detail::getHostCPUNameForBPF() {
355367
} attr = {};
356368
attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
357369
attr.insn_cnt = 5;
358-
attr.insns = (uint64_t)insns;
370+
attr.insns = (uint64_t)v3_insns;
359371
attr.license = (uint64_t)"DUMMY";
360372

361-
int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr));
373+
int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr,
374+
sizeof(attr));
375+
if (fd >= 0) {
376+
close(fd);
377+
return "v3";
378+
}
379+
380+
/* Clear the whole attr in case its content changed by syscall. */
381+
memset(&attr, 0, sizeof(attr));
382+
attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
383+
attr.insn_cnt = 5;
384+
attr.insns = (uint64_t)v2_insns;
385+
attr.license = (uint64_t)"DUMMY";
386+
fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr));
362387
if (fd >= 0) {
363388
close(fd);
364389
return "v2";

‎llvm/lib/Target/BPF/BPF.td

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class Proc<string Name, list<SubtargetFeature> Features>
2020
def : Proc<"generic", []>;
2121
def : Proc<"v1", []>;
2222
def : Proc<"v2", []>;
23+
def : Proc<"v3", []>;
2324
def : Proc<"probe", []>;
2425

2526
def DummyFeature : SubtargetFeature<"dummy", "isDummyMode",

‎llvm/lib/Target/BPF/BPFISelLowering.cpp

+23-34
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
105105

106106
if (STI.getHasAlu32()) {
107107
setOperationAction(ISD::BSWAP, MVT::i32, Promote);
108-
setOperationAction(ISD::BR_CC, MVT::i32, Promote);
108+
setOperationAction(ISD::BR_CC, MVT::i32,
109+
STI.getHasJmp32() ? Custom : Promote);
109110
}
110111

111112
setOperationAction(ISD::CTTZ, MVT::i64, Custom);
@@ -162,6 +163,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
162163

163164
// CPU/Feature control
164165
HasAlu32 = STI.getHasAlu32();
166+
HasJmp32 = STI.getHasJmp32();
165167
HasJmpExt = STI.getHasJmpExt();
166168
}
167169

@@ -506,7 +508,7 @@ SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
506508
NegateCC(LHS, RHS, CC);
507509

508510
return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
509-
DAG.getConstant(CC, DL, MVT::i64), Dest);
511+
DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
510512
}
511513

512514
SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
@@ -676,36 +678,23 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
676678
int CC = MI.getOperand(3).getImm();
677679
int NewCC;
678680
switch (CC) {
679-
case ISD::SETGT:
680-
NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri;
681-
break;
682-
case ISD::SETUGT:
683-
NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri;
684-
break;
685-
case ISD::SETGE:
686-
NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri;
687-
break;
688-
case ISD::SETUGE:
689-
NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri;
690-
break;
691-
case ISD::SETEQ:
692-
NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri;
693-
break;
694-
case ISD::SETNE:
695-
NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri;
696-
break;
697-
case ISD::SETLT:
698-
NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri;
699-
break;
700-
case ISD::SETULT:
701-
NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri;
702-
break;
703-
case ISD::SETLE:
704-
NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri;
705-
break;
706-
case ISD::SETULE:
707-
NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri;
708-
break;
681+
#define SET_NEWCC(X, Y) \
682+
case ISD::X: \
683+
if (is32BitCmp && HasJmp32) \
684+
NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
685+
else \
686+
NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
687+
break
688+
SET_NEWCC(SETGT, JSGT);
689+
SET_NEWCC(SETUGT, JUGT);
690+
SET_NEWCC(SETGE, JSGE);
691+
SET_NEWCC(SETUGE, JUGE);
692+
SET_NEWCC(SETEQ, JEQ);
693+
SET_NEWCC(SETNE, JNE);
694+
SET_NEWCC(SETLT, JSLT);
695+
SET_NEWCC(SETULT, JULT);
696+
SET_NEWCC(SETLE, JSLE);
697+
SET_NEWCC(SETULE, JULE);
709698
default:
710699
report_fatal_error("unimplemented select CondCode " + Twine(CC));
711700
}
@@ -723,13 +712,13 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
723712
//
724713
// We simply do extension for all situations in this method, but we will
725714
// try to remove those unnecessary in BPFMIPeephole pass.
726-
if (is32BitCmp)
715+
if (is32BitCmp && !HasJmp32)
727716
LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
728717

729718
if (isSelectRROp) {
730719
unsigned RHS = MI.getOperand(2).getReg();
731720

732-
if (is32BitCmp)
721+
if (is32BitCmp && !HasJmp32)
733722
RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
734723

735724
BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);

‎llvm/lib/Target/BPF/BPFISelLowering.h

+2
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ class BPFTargetLowering : public TargetLowering {
5555
MachineBasicBlock *BB) const override;
5656

5757
bool getHasAlu32() const { return HasAlu32; }
58+
bool getHasJmp32() const { return HasJmp32; }
5859
bool getHasJmpExt() const { return HasJmpExt; }
5960

6061
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
@@ -65,6 +66,7 @@ class BPFTargetLowering : public TargetLowering {
6566
private:
6667
// Control Instruction Selection Features
6768
bool HasAlu32;
69+
bool HasJmp32;
6870
bool HasJmpExt;
6971

7072
SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;

‎llvm/lib/Target/BPF/BPFInstrFormats.td

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ def BPF_ST : BPFOpClass<0x2>;
1616
def BPF_STX : BPFOpClass<0x3>;
1717
def BPF_ALU : BPFOpClass<0x4>;
1818
def BPF_JMP : BPFOpClass<0x5>;
19+
def BPF_JMP32 : BPFOpClass<0x6>;
1920
def BPF_ALU64 : BPFOpClass<0x7>;
2021

2122
class BPFSrcType<bits<1> val> {

‎llvm/lib/Target/BPF/BPFInstrInfo.td

+65-11
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,26 @@ def BPF_CC_LTU : PatLeaf<(i64 imm),
101101
[{return (N->getZExtValue() == ISD::SETULT);}]>;
102102
def BPF_CC_LEU : PatLeaf<(i64 imm),
103103
[{return (N->getZExtValue() == ISD::SETULE);}]>;
104+
def BPF_CC_EQ_32 : PatLeaf<(i32 imm),
105+
[{return (N->getZExtValue() == ISD::SETEQ);}]>;
106+
def BPF_CC_NE_32 : PatLeaf<(i32 imm),
107+
[{return (N->getZExtValue() == ISD::SETNE);}]>;
108+
def BPF_CC_GE_32 : PatLeaf<(i32 imm),
109+
[{return (N->getZExtValue() == ISD::SETGE);}]>;
110+
def BPF_CC_GT_32 : PatLeaf<(i32 imm),
111+
[{return (N->getZExtValue() == ISD::SETGT);}]>;
112+
def BPF_CC_GTU_32 : PatLeaf<(i32 imm),
113+
[{return (N->getZExtValue() == ISD::SETUGT);}]>;
114+
def BPF_CC_GEU_32 : PatLeaf<(i32 imm),
115+
[{return (N->getZExtValue() == ISD::SETUGE);}]>;
116+
def BPF_CC_LE_32 : PatLeaf<(i32 imm),
117+
[{return (N->getZExtValue() == ISD::SETLE);}]>;
118+
def BPF_CC_LT_32 : PatLeaf<(i32 imm),
119+
[{return (N->getZExtValue() == ISD::SETLT);}]>;
120+
def BPF_CC_LTU_32 : PatLeaf<(i32 imm),
121+
[{return (N->getZExtValue() == ISD::SETULT);}]>;
122+
def BPF_CC_LEU_32 : PatLeaf<(i32 imm),
123+
[{return (N->getZExtValue() == ISD::SETULE);}]>;
104124

105125
// For arithmetic and jump instructions the 8-bit 'code'
106126
// field is divided into three parts:
@@ -166,23 +186,57 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
166186
let BPFClass = BPF_JMP;
167187
}
168188

169-
multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> {
189+
class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
190+
: TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
191+
(outs),
192+
(ins GPR32:$dst, GPR32:$src, brtarget:$BrDst),
193+
"if $dst "#OpcodeStr#" $src goto $BrDst",
194+
[(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> {
195+
bits<4> dst;
196+
bits<4> src;
197+
bits<16> BrDst;
198+
199+
let Inst{55-52} = src;
200+
let Inst{51-48} = dst;
201+
let Inst{47-32} = BrDst;
202+
let BPFClass = BPF_JMP32;
203+
}
204+
205+
class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
206+
: TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
207+
(outs),
208+
(ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst),
209+
"if $dst "#OpcodeStr#" $imm goto $BrDst",
210+
[(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> {
211+
bits<4> dst;
212+
bits<16> BrDst;
213+
bits<32> imm;
214+
215+
let Inst{51-48} = dst;
216+
let Inst{47-32} = BrDst;
217+
let Inst{31-0} = imm;
218+
let BPFClass = BPF_JMP32;
219+
}
220+
221+
multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> {
170222
def _rr : JMP_RR<Opc, OpcodeStr, Cond>;
171223
def _ri : JMP_RI<Opc, OpcodeStr, Cond>;
224+
def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>;
225+
def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>;
172226
}
173227

174228
let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in {
175229
// cmp+goto instructions
176-
defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ>;
177-
defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU>;
178-
defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU>;
179-
defm JNE : J<BPF_JNE, "!=", BPF_CC_NE>;
180-
defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT>;
181-
defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE>;
182-
defm JULT : J<BPF_JLT, "<", BPF_CC_LTU>;
183-
defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU>;
184-
defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT>;
185-
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE>;
230+
defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>;
231+
defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>;
232+
defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>;
233+
defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>;
234+
defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>;
235+
defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>;
236+
defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>;
237+
defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>;
238+
defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
239+
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
186240
}
187241

188242
// ALU instructions

‎llvm/lib/Target/BPF/BPFSubtarget.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ BPFSubtarget &BPFSubtarget::initializeSubtargetDependencies(StringRef CPU,
3535

3636
void BPFSubtarget::initializeEnvironment() {
3737
HasJmpExt = false;
38+
HasJmp32 = false;
3839
HasAlu32 = false;
3940
UseDwarfRIS = false;
4041
}
@@ -48,6 +49,11 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
4849
HasJmpExt = true;
4950
return;
5051
}
52+
if (CPU == "v3") {
53+
HasJmpExt = true;
54+
HasJmp32 = true;
55+
return;
56+
}
5157
}
5258

5359
BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU,

‎llvm/lib/Target/BPF/BPFSubtarget.h

+5
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
4747
// whether the cpu supports jmp ext
4848
bool HasJmpExt;
4949

50+
// whether the cpu supports jmp32 ext.
51+
// NOTE: jmp32 is not enabled when alu32 enabled.
52+
bool HasJmp32;
53+
5054
// whether the cpu supports alu32 instructions.
5155
bool HasAlu32;
5256

@@ -65,6 +69,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
6569
// subtarget options. Definition of function is auto generated by tblgen.
6670
void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
6771
bool getHasJmpExt() const { return HasJmpExt; }
72+
bool getHasJmp32() const { return HasJmp32; }
6873
bool getHasAlu32() const { return HasAlu32; }
6974
bool getUseDwarfRIS() const { return UseDwarfRIS; }
7075

‎llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class BPFDisassembler : public MCDisassembler {
3939
BPF_STX = 0x3,
4040
BPF_ALU = 0x4,
4141
BPF_JMP = 0x5,
42-
BPF_RES = 0x6,
42+
BPF_JMP32 = 0x6,
4343
BPF_ALU64 = 0x7
4444
};
4545

‎llvm/test/MC/BPF/insn-unit-32.s

+47
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,50 @@
5454
// CHECK: b4 09 00 00 01 00 00 00 w9 = 1
5555
// CHECK: b4 09 00 00 ff ff ff ff w9 = -1
5656
// CHECK: c4 0a 00 00 40 00 00 00 w10 s>>= 64
57+
58+
if w0 == w1 goto Llabel0 // BPF_JEQ | BPF_X
59+
if w3 != w4 goto Llabel0 // BPF_JNE | BPF_X
60+
// CHECK: 1e 10 0b 00 00 00 00 00 if w0 == w1 goto +11
61+
// CHECK: 5e 43 0a 00 00 00 00 00 if w3 != w4 goto +10
62+
63+
if w1 > w2 goto Llabel0 // BPF_JGT | BPF_X
64+
if w2 >= w3 goto Llabel0 // BPF_JGE | BPF_X
65+
if w4 s> w5 goto Llabel0 // BPF_JSGT | BPF_X
66+
if w5 s>= w6 goto Llabel0 // BPF_JSGE | BPF_X
67+
// CHECK: 2e 21 09 00 00 00 00 00 if w1 > w2 goto +9
68+
// CHECK: 3e 32 08 00 00 00 00 00 if w2 >= w3 goto +8
69+
// CHECK: 6e 54 07 00 00 00 00 00 if w4 s> w5 goto +7
70+
// CHECK: 7e 65 06 00 00 00 00 00 if w5 s>= w6 goto +6
71+
72+
if w6 < w7 goto Llabel0 // BPF_JLT | BPF_X
73+
if w7 <= w8 goto Llabel0 // BPF_JLE | BPF_X
74+
if w8 s< w9 goto Llabel0 // BPF_JSLT | BPF_X
75+
if w9 s<= w10 goto Llabel0 // BPF_JSLE | BPF_X
76+
// CHECK: ae 76 05 00 00 00 00 00 if w6 < w7 goto +5
77+
// CHECK: be 87 04 00 00 00 00 00 if w7 <= w8 goto +4
78+
// CHECK: ce 98 03 00 00 00 00 00 if w8 s< w9 goto +3
79+
// CHECK: de a9 02 00 00 00 00 00 if w9 s<= w10 goto +2
80+
81+
if w0 == 0 goto Llabel0 // BPF_JEQ | BPF_K
82+
if w3 != -1 goto Llabel0 // BPF_JNE | BPF_K
83+
// CHECK: 16 00 01 00 00 00 00 00 if w0 == 0 goto +1
84+
// CHECK: 56 03 00 00 ff ff ff ff if w3 != -1 goto +0
85+
86+
Llabel0 :
87+
if w1 > 64 goto Llabel0 // BPF_JGT | BPF_K
88+
if w2 >= 0xffffffff goto Llabel0 // BPF_JGE | BPF_K
89+
if w4 s> 0xffffffff goto Llabel0 // BPF_JSGT | BPF_K
90+
if w5 s>= 0x7fffffff goto Llabel0 // BPF_JSGE | BPF_K
91+
// CHECK: 26 01 ff ff 40 00 00 00 if w1 > 64 goto -1
92+
// CHECK: 36 02 fe ff ff ff ff ff if w2 >= -1 goto -2
93+
// CHECK: 66 04 fd ff ff ff ff ff if w4 s> -1 goto -3
94+
// CHECK: 76 05 fc ff ff ff ff 7f if w5 s>= 2147483647 goto -4
95+
96+
if w6 < 0xff goto Llabel0 // BPF_JLT | BPF_K
97+
if w7 <= 0xffff goto Llabel0 // BPF_JLE | BPF_K
98+
if w8 s< 0 goto Llabel0 // BPF_JSLT | BPF_K
99+
if w9 s<= -1 goto Llabel0 // BPF_JSLE | BPF_K
100+
// CHECK: a6 06 fb ff ff 00 00 00 if w6 < 255 goto -5
101+
// CHECK: b6 07 fa ff ff ff 00 00 if w7 <= 65535 goto -6
102+
// CHECK: c6 08 f9 ff 00 00 00 00 if w8 s< 0 goto -7
103+
// CHECK: d6 09 f8 ff ff ff ff ff if w9 s<= -1 goto -8

0 commit comments

Comments
 (0)
Please sign in to comment.