Patchwork Netdev + BPF  Patches  Bundles  About this project Login Register Mail settings

[bpf-next] riscv, bpf: Implement more atomic operations for RV64 12808103 diff mbox series

Message ID 20220410101246.232875-1-pulehui@huawei.com (mailing list archive)

State Accepted
Commit dd642ccb45ecce1402eb2550f5284fc6bb9ed7b8
Delegated to: BPF
Headers show
Series [bpf-next] riscv, bpf: Implement more atomic operations for RV64 | expand

Checks
Context Check Description

netdev/tree_selection success Clearly marked for bpf-next

netdev/fixes_present success Fixes tag not required for -next series

netdev/subject_prefix success Link

netdev/cover_letter success Single patches do not need cover letters

netdev/patch_count success Link

netdev/header_inline success No static functions without inline keyword in header files

netdev/build_32bit success Errors and warnings before: 0 this patch: 0

netdev/cc_maintainers warning 4 maintainers not CCed: aou@eecs.berkeley.edu palmer@dabbelt.com paul.walmsley@sifive.com netdev@vger.kernel.org

netdev/build_clang success Errors and warnings before: 0 this patch: 0

netdev/module_param success Was 0 now: 0

netdev/verify_signedoff success Signed-off-by tag matches author and committer

netdev/verify_fixes success No Fixes tag

netdev/build_allmodconfig_warn success Errors and warnings before: 0 this patch: 0

netdev/checkpatch success total: 0 errors, 0 warnings, 0 checks, 201 lines checked

netdev/kdoc success Errors and warnings before: 0 this patch: 0

netdev/source_inline success Was 0 now: 0

bpf/vmtest-bpf-next-PR success PR summary

bpf/vmtest-bpf-next-VM_Test-1 success Logs for Kernel LATEST on ubuntu-latest + selftests

bpf/vmtest-bpf-next-VM_Test-2 success Logs for Kernel LATEST on z15 + selftests

Commit Message
Pu Lehui April 10, 2022, 10:12 a.m. UTC

This patch implement more bpf atomic operations for RV64.

The added operations are shown below:

atomic[64]_[fetch_]add
atomic[64]_[fetch_]and
atomic[64]_[fetch_]or
atomic[64]_xchg
atomic[64]_cmpxchg

Since riscv specification does not provide AMO instruction for

CAS operation, we use lr/sc instruction for cmpxchg operation,
and AMO instructions for the rest ops. Tests "test_bpf.ko" and
"test_progs -t atomic" have passed, as well as "test_verifier"
with no new failure ceses.

Signed-off-by: Pu Lehui <pulehui@huawei.com>

---
arch/riscv/net/bpf_jit.h | 67 +++++++++++++++++++
arch/riscv/net/bpf_jit_comp64.c | 110 +++++++++++++++++++++++++-------
2 files changed, 153 insertions(+), 24 deletions(-)

Comments
Song Liu April 11, 2022, 5:44 a.m. UTC | #1

> On Apr 10, 2022, at 3:12 AM, Pu Lehui <pulehui@huawei.com> wrote:

>
> This patch implement more bpf atomic operations for RV64.
> The added operations are shown below:
>
> atomic[64]_[fetch_]add
> atomic[64]_[fetch_]and
> atomic[64]_[fetch_]or
> atomic[64]_xchg
> atomic[64]_cmpxchg
>
> Since riscv specification does not provide AMO instruction for
> CAS operation, we use lr/sc instruction for cmpxchg operation,
> and AMO instructions for the rest ops. Tests "test_bpf.ko" and
> "test_progs -t atomic" have passed, as well as "test_verifier"
> with no new failure ceses.
>
> Signed-off-by: Pu Lehui <pulehui@huawei.com>

LGTM.

Acked-by: Song Liu <songliubraving@fb.com>

Björn Töpel April 11, 2022, 12:49 p.m. UTC | #2

On Sun, 10 Apr 2022 at 11:43, Pu Lehui <pulehui@huawei.com> wrote:

>
> This patch implement more bpf atomic operations for RV64.
> The added operations are shown below:
>
> atomic[64]_[fetch_]add
> atomic[64]_[fetch_]and
> atomic[64]_[fetch_]or
> atomic[64]_xchg
> atomic[64]_cmpxchg
>
> Since riscv specification does not provide AMO instruction for
> CAS operation, we use lr/sc instruction for cmpxchg operation,
> and AMO instructions for the rest ops. Tests "test_bpf.ko" and
> "test_progs -t atomic" have passed, as well as "test_verifier"
> with no new failure ceses.
>
> Signed-off-by: Pu Lehui <pulehui@huawei.com>

Thank you for implementing this! It's been long overdue.

Acked-by: Björn Töpel <bjorn@kernel.org>

patchwork-bot+netdevbpf@kernel.org April 11, 2022, 3 p.m. UTC | #3

Hello:

This patch was applied to bpf/bpf-next.git (master)

by Daniel Borkmann <daniel@iogearbox.net>:

On Sun, 10 Apr 2022 18:12:46 +0800 you wrote:

> This patch implement more bpf atomic operations for RV64.
> The added operations are shown below:
>
> atomic[64]_[fetch_]add
> atomic[64]_[fetch_]and
> atomic[64]_[fetch_]or
> atomic[64]_xchg
> atomic[64]_cmpxchg
>
> [...]

Here is the summary with links:

- [bpf-next] riscv, bpf: Implement more atomic operations for RV64
https://git.kernel.org/bpf/bpf-next/c/dd642ccb45ec

You are awesome, thank you!

Patch 12808103 diff mbox series

diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h

index f42d9cd3b..2a3715bf2 100644
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -535,6 +535,43 @@ static inline u32 rv_amoadd_w(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
return rv_amo_insn(0,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
}

+static inline u32 rv_amoand_w(u8

amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0xc,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_amoor_w(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x8,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_amoxor_w(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x4,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_amoswap_w(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x1,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x2,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x3,
amo aq, rl, rs2, rs1, 2, rd, 0x2f);
+}
+
+static inline u32 rv_fence(u8 pred, u8 succ)
+{
+ u16 imm11_0 = pred << 4 | succ;
+
+ return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
+}
+
/* RVC instrutions. */

static inline u16 rvc_addi4spn(u8 rd, u32 imm10)

@@ -753,6 +790,36 @@ static inline u32 rv_amoadd_d(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
return rv_amo_insn(0,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
}

+static inline u32 rv_amoand_d(u8

amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0xc,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
+static inline u32 rv_amoor_d(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x8,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
+static inline u32 rv_amoxor_d(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x4,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
+static inline u32 rv_amoswap_d(u8
amo rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x1,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
+static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x2,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
+static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
+{
+ return rv_amo_insn(0x3,
amo aq, rl, rs2, rs1, 3, rd, 0x2f);
+}
+
/* RV64-only RVC instructions. */

static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 0bcda99d1..00df3a8f9 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -455,6 +455,90 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
return 0;
}

+static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
+ struct rv_jit_context *ctx)
+{
+ u8 r0;
+ int jmp_offset;
+
+ if (off) {
+ if (is_12b_int(off)) {
+ emit_addi(RV_REG_T1, rd, off, ctx);
+ } else {
+ emit_imm(RV_REG_T1, off, ctx);
+ emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
+ }
+ rd = RV_REG_T1;
+ }
+
+ switch (imm) {
+ /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */
+ case BPF_ADD:
+ emit(is64 ? rv_amoadd_d(RV_REG_ZERO,
amo rs, rd, 0, 0) :
+ rv_amoadd_w(RV_REG_ZERO,
amo rs, rd, 0, 0), ctx);
+ break;
+ case BPF_AND:
+ emit(is64 ? rv_amoand_d(RV_REG_ZERO,
amo rs, rd, 0, 0) :
+ rv_amoand_w(RV_REG_ZERO,
amo rs, rd, 0, 0), ctx);
+ break;
+ case BPF_OR:
+ emit(is64 ? rv_amoor_d(RV_REG_ZERO,
amo rs, rd, 0, 0) :
+ rv_amoor_w(RV_REG_ZERO,
amo rs, rd, 0, 0), ctx);
+ break;
+ case BPF_XOR:
+ emit(is64 ? rv_amoxor_d(RV_REG_ZERO,
amo rs, rd, 0, 0) :
+ rv_amoxor_w(RV_REG_ZERO,
amo rs, rd, 0, 0), ctx);
+ break;
+ /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
+ case BPF_ADD | BPF_FETCH:
+ emit(is64 ? rv_amoadd_d(rs,
amo rs, rd, 0, 0) :
+ rv_amoadd_w(rs,
amo rs, rd, 0, 0), ctx);
+ if (!is64)
+ emit_zext_32(rs, ctx);
+ break;
+ case BPF_AND | BPF_FETCH:
+ emit(is64 ? rv_amoand_d(rs,
amo rs, rd, 0, 0) :
+ rv_amoand_w(rs,
amo rs, rd, 0, 0), ctx);
+ if (!is64)
+ emit_zext_32(rs, ctx);
+ break;
+ case BPF_OR | BPF_FETCH:
+ emit(is64 ? rv_amoor_d(rs,
amo rs, rd, 0, 0) :
+ rv_amoor_w(rs,
amo rs, rd, 0, 0), ctx);
+ if (!is64)
+ emit_zext_32(rs, ctx);
+ break;
+ case BPF_XOR | BPF_FETCH:
+ emit(is64 ? rv_amoxor_d(rs,
amo rs, rd, 0, 0) :
+ rv_amoxor_w(rs,
amo rs, rd, 0, 0), ctx);
+ if (!is64)
+ emit_zext_32(rs, ctx);
+ break;
+ /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
+ case BPF_XCHG:
+ emit(is64 ? rv_amoswap_d(rs,
amo rs, rd, 0, 0) :
+ rv_amoswap_w(rs,
amo rs, rd, 0, 0), ctx);
+ if (!is64)
+ emit_zext_32(rs, ctx);
+ break;
+ /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
+ case BPF_CMPXCHG:
+ r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
+ emit(is64 ? rv_addi(RV_REG_T2, r0, 0) :
+ rv_addiw(RV_REG_T2, r0, 0), ctx);
+ emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
+ rv_lr_w(r0, 0, rd, 0, 0), ctx);
+ jmp_offset = ninsns_rvoff(8);
+ emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
+ emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) :
+ rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx);
+ jmp_offset = ninsns_rvoff(-6);
+ emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
+ emit(rv_fence(0x3, 0x3), ctx);
+ break;
+ }
+}
+
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
#define BPF_FIXUP_REG_MASK GENMASK(31, 27)

@@ -1146,30 +1230,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
break;
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
- if (insn->imm != BPF_ADD) {
- pr_err("bpf-jit: not supported: atomic operation %02x ***\n",
- insn->imm);
- return -EINVAL;
- }
-
- /* atomic_add: lock *(u32 *)(dst + off) += src
- * atomic_add: lock *(u64 *)(dst + off) += src
- */
-
- if (off) {
- if (is_12b_int(off)) {
- emit_addi(RV_REG_T1, rd, off, ctx);
- } else {
- emit_imm(RV_REG_T1, off, ctx);
- emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
- }
-
- rd = RV_REG_T1;
- }
-
- emit(BPF_SIZE(code) == BPF_W ?
- rv_amoadd_w(RV_REG_ZERO,
amo rs, rd, 0, 0) :
- rv_amoadd_d(RV_REG_ZERO,
amo rs, rd, 0, 0), ctx);
+ emit_atomic(rd, rs, off, imm,
+ BPF_SIZE(code) == BPF_DW, ctx);
break;
default:
pr_err("bpf-jit: unknown opcode %02x\n", code);

patchwork patch tracking system | version v2.2.6 | about patchwork