在 2023/10/9 12:35, Yi Zhang 写道:
Hello
blktests srp lead kernel panic[2] on aarch64 when the kernel enabled
CONFIG_ARM64_64K_PAGES, bisect shows it was introduced from commit[1],
pls help check it and let me know if you need any info/testing for it, thanks.
[1]
commit 325a7eb85199ec9c5b5a7af812f43ea16b735569
Author: Bob Pearson <rpearsonhpe@xxxxxxxxx>
Date: Thu Jan 19 17:59:36 2023 -0600
RDMA/rxe: Cleanup page variables in rxe_mr.c
Cleanup usage of mr->page_shift and mr->page_mask and introduce
an extractor for mr->ibmr.page_size. Normal usage in the kernel
has page_mask masking out offset in page rather than masking out
the page number. The rxe driver had reversed that which was confusing.
Implicitly there can be a per mr page_size which was not uniformly
supported.
Link: https://lore.kernel.org/r/20230119235936.19728-6-rpearsonhpe@xxxxxxxxx
Signed-off-by: Bob Pearson <rpearsonhpe@xxxxxxxxx>
Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
Hi, Yi
I delved into the commit. And the commit can not be reverted cleanly. So
I made the following diff to try to revert this commit. After this
commit is applied, rping can work well.
But I can not reproduce the problem that you reported. Can you try to
make tests with the following diff to verify if this problem is fixed or
not? Thanks a lot.
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c
b/drivers/infiniband/sw/rxe/rxe_mr.c
index f54042e9aeb2..50a58fdd12f0 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -58,9 +58,6 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
mr->rkey = mr->ibmr.rkey = key;
mr->access = access;
- mr->ibmr.page_size = PAGE_SIZE;
- mr->page_mask = PAGE_MASK;
- mr->page_shift = PAGE_SHIFT;
mr->state = RXE_MR_STATE_INVALID;
}
@@ -79,7 +76,7 @@ static unsigned long rxe_mr_iova_to_index(struct
rxe_mr *mr, u64 iova)
static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64
iova)
{
- return iova & (mr_page_size(mr) - 1);
+ return iova & (PAGE_SIZE - 1);
}
static bool is_pmem_page(struct page *pg)
@@ -143,6 +140,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start,
u64 length, u64 iova,
return PTR_ERR(umem);
}
+ mr->page_shift = PAGE_SHIFT;
+ mr->page_mask = PAGE_SIZE - 1;
+
err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
if (err) {
ib_umem_release(umem);
@@ -232,14 +232,16 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct
scatterlist *sgl,
int sg_nents, unsigned int *sg_offset)
{
struct rxe_mr *mr = to_rmr(ibmr);
- unsigned int page_size = mr_page_size(mr);
+ int n;
mr->nbuf = 0;
- mr->page_shift = ilog2(page_size);
- mr->page_mask = ~((u64)page_size - 1);
- mr->page_offset = mr->ibmr.iova & (page_size - 1);
+ n = ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
+
+ mr->page_shift = ilog2(ibmr->page_size);
+ mr->page_mask = ibmr->page_size - 1;
+ mr->offset = ibmr->iova & mr->page_mask;
- return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
+ return n;
}
static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h
b/drivers/infiniband/sw/rxe/rxe_verbs.h
index ccb9d19ffe8a..f1424d315acd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -309,7 +309,7 @@ struct rxe_mr {
int access;
atomic_t num_mw;
- unsigned int page_offset;
+ u32 offset;
unsigned int page_shift;
u64 page_mask;
@@ -321,7 +321,7 @@ struct rxe_mr {
static inline unsigned int mr_page_size(struct rxe_mr *mr)
{
- return mr ? mr->ibmr.page_size : PAGE_SIZE;
+ return PAGE_SIZE;
}
enum rxe_mw_state {
Zhu Yanjun
[2] dmesg:
[ 1120.381103] run blktests srp/001 at 2023-10-08 21:02:34
[ 1120.647692] null_blk: module loaded
[ 1120.675092] null_blk: disk nullb0 created
[ 1120.683950] null_blk: disk nullb1 created
[ 1121.053881] rdma_rxe: loaded
[ 1121.071624] (null): rxe_set_mtu: Set mtu to 1024
[ 1121.080184] infiniband enP2p1s0v0_rxe: set active
[ 1121.080194] infiniband enP2p1s0v0_rxe: added enP2p1s0v0
[ 1121.128323] scsi_debug:sdebug_add_store: dif_storep 524288 bytes @
ffff80009a460000
[ 1121.129215] scsi_debug:sdebug_driver_probe: scsi_debug: trim
poll_queues to 0. poll_q/nr_hw = (0/1)
[ 1121.138297] scsi_debug:sdebug_driver_probe: host protection DIF3 DIX3
[ 1121.138310] scsi host4: scsi_debug: version 0191 [20210520]
dev_size_mb=32, opts=0x0, submit_queues=1, statistics=0
[ 1121.138813] scsi 4:0:0:0: Direct-Access Linux scsi_debug
0191 PQ: 0 ANSI: 7
[ 1121.139134] scsi 4:0:0:0: Power-on or device reset occurred
[ 1121.145365] sd 4:0:0:0: Attached scsi generic sg1 type 0
[ 1121.145439] sd 4:0:0:0: [sdb] 65536 512-byte logical blocks: (33.6
MB/32.0 MiB)
[ 1121.145501] sd 4:0:0:0: [sdb] Write Protect is off
[ 1121.145513] sd 4:0:0:0: [sdb] Mode Sense: 73 00 10 08
[ 1121.145601] sd 4:0:0:0: [sdb] Write cache: enabled, read cache:
enabled, supports DPO and FUA
[ 1121.145758] sd 4:0:0:0: [sdb] Enabling DIX T10-DIF-TYPE3-CRC,
application tag size 6 bytes
[ 1121.145769] sd 4:0:0:0: [sdb] Enabling DIF Type 3 protection
[ 1121.145777] sd 4:0:0:0: [sdb] Preferred minimum I/O size 512 bytes
[ 1121.145784] sd 4:0:0:0: [sdb] Optimal transfer size 524288 bytes
[ 1121.148475] sd 4:0:0:0: [sdb] Attached SCSI disk
[ 1121.552336] Rounding down aligned max_sectors from 4294967295 to 4294967168
[ 1121.602229] ib_srpt:srpt_add_one: ib_srpt device = 000000000a5842dc
[ 1121.602256] ib_srpt:srpt_use_srq: ib_srpt
srpt_use_srq(enP2p1s0v0_rxe): use_srq = 0; ret = 0
[ 1121.602266] ib_srpt:srpt_add_one: ib_srpt Target login info:
id_ext=1e1b0dfffe9f67ae,ioc_guid=1e1b0dfffe9f67ae,pkey=ffff,service_id=1e1b0dfffe9f67ae
[ 1121.602290] ib_srpt:srpt_add_one: ib_srpt added enP2p1s0v0_rxe.
[ 1121.920615] Rounding down aligned max_sectors from 255 to 128
[ 1121.948272] Rounding down aligned max_sectors from 255 to 128
[ 1121.975917] Rounding down aligned max_sectors from 4294967295 to 4294967168
[ 1122.487117] ib_srp:srp_add_one: ib_srp: srp_add_one:
18446744073709551615 / 4096 = 4503599627370495 <> 512
[ 1122.487136] ib_srp:srp_add_one: ib_srp: enP2p1s0v0_rxe:
mr_page_shift = 12, device->max_mr_size = 0xffffffffffffffff,
device->max_fast_reg_page_list_len = 512, max_pages_per_mr = 512,
mr_max_size = 0x200000
[ 1122.539669] ib_srp:srp_parse_in: ib_srp: 10.19.240.81 -> 10.19.240.81:0
[ 1122.539691] ib_srp:srp_parse_in: ib_srp: 10.19.240.81:5555 ->
10.19.240.81:5555
[ 1122.539700] ib_srp:add_target_store: ib_srp: max_sectors = 1024;
max_pages_per_mr = 512; mr_page_size = 4096; max_sectors_per_mr =
4096; mr_per_cmd = 2
[ 1122.539710] ib_srp:srp_max_it_iu_len: ib_srp: max_iu_len = 8260
[ 1122.544979] ib_srpt Received SRP_LOGIN_REQ with i_port_id
fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae, t_port_id
1e1b:0dff:fe9f:67ae:1e1b:0dff:fe9f:67ae and it_iu_len 8260 on port 1
(guid=fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae); pkey 0xffff
[ 1122.545168] ib_srpt:srpt_cm_req_recv: ib_srpt imm_data_offset = 68
[ 1122.546622] ib_srpt:srpt_create_ch_ib: ib_srpt srpt_create_ch_ib:
max_cqe= 8191 max_sge= 32 sq_size = 8191 ch= 000000007ddb8927
[ 1122.546726] ib_srpt:srpt_cm_req_recv: ib_srpt registering src addr
10.19.240.81 or i_port_id 0xfe800000000000001e1b0dfffe9f67ae
[ 1122.546780] ib_srpt:srpt_cm_req_recv: ib_srpt Establish connection
sess=00000000f393b6f7 name=10.19.240.81 ch=000000007ddb8927
[ 1122.546973] ib_srp:srp_max_it_iu_len: ib_srp: max_iu_len = 8260
[ 1122.546988] scsi host5: ib_srp: using immediate data
[ 1122.547044] ib_srpt:srpt_zerolength_write: ib_srpt 10.19.240.81-18:
queued zerolength write
[ 1122.547127] ib_srpt:srpt_zerolength_write_done: ib_srpt
10.19.240.81-18 wc->status 0
[ 1122.552276] ib_srpt Received SRP_LOGIN_REQ with i_port_id
fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae, t_port_id
1e1b:0dff:fe9f:67ae:1e1b:0dff:fe9f:67ae and it_iu_len 8260 on port 1
(guid=fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae); pkey 0xffff
[ 1122.552527] ib_srpt:srpt_cm_req_recv: ib_srpt imm_data_offset = 68
[ 1122.552969] ib_srpt:srpt_create_ch_ib: ib_srpt srpt_create_ch_ib:
max_cqe= 8191 max_sge= 32 sq_size = 8191 ch= 00000000080908b6
[ 1122.553054] ib_srpt:srpt_cm_req_recv: ib_srpt registering src addr
10.19.240.81 or i_port_id 0xfe800000000000001e1b0dfffe9f67ae
[ 1122.553095] ib_srpt:srpt_cm_req_recv: ib_srpt Establish connection
sess=000000005db9b14f name=10.19.240.81 ch=00000000080908b6
[ 1122.553204] ib_srp:srp_max_it_iu_len: ib_srp: max_iu_len = 8260
[ 1122.553216] scsi host5: ib_srp: using immediate data
[ 1122.553267] ib_srpt:srpt_zerolength_write: ib_srpt 10.19.240.81-20:
queued zerolength write
[ 1122.553346] ib_srpt:srpt_zerolength_write_done: ib_srpt
10.19.240.81-20 wc->status 0
[ 1122.558431] ib_srpt Received SRP_LOGIN_REQ with i_port_id
fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae, t_port_id
1e1b:0dff:fe9f:67ae:1e1b:0dff:fe9f:67ae and it_iu_len 8260 on port 1
(guid=fe80:0000:0000:0000:1e1b:0dff:fe9f:67ae); pkey 0xffff
[ 1122.558665] ib_srpt:srpt_cm_req_recv: ib_srpt imm_data_offset = 68
[ 1122.559112] ib_srpt:srpt_create_ch_ib: ib_srpt srpt_create_ch_ib:
max_cqe= 8191 max_sge= 32 sq_size = 8191 ch= 000000005bf29d9f
[ 1122.559196] ib_srpt:srpt_cm_req_recv: ib_srpt registering src addr
10.19.240.81 or i_port_id 0xfe800000000000001e1b0dfffe9f67ae
[ 1122.559235] ib_srpt:srpt_cm_req_recv: ib_srpt Establish connection
sess=00000000496e5c2f name=10.19.240.81 ch=000000005bf29d9f
[ 1122.559342] ib_srp:srp_max_it_iu_len: ib_srp: max_iu_len = 8260
[ 1122.559352] scsi host5: ib_srp: using immediate data
[ 1122.559390] Unable to handle kernel paging request at virtual
address 001bb829281c2fb8
[ 1122.559406] ib_srpt:srpt_zerolength_write: ib_srpt 10.19.240.81-22:
queued zerolength write
[ 1122.567320] Mem abort info:
[ 1122.567323] ESR = 0x0000000096000004
[ 1122.567326] EC = 0x25: DABT (current EL), IL = 32 bits
[ 1122.567330] SET = 0, FnV = 0
[ 1122.567332] EA = 0, S1PTW = 0
[ 1122.567335] FSC = 0x04: level 0 translation fault
[ 1122.567338] Data abort info:
[ 1122.567340] ISV = 0, ISS = 0x00000004, ISS2 = 0x00000000
[ 1122.567343] CM = 0, WnR = 0, TnD = 0, TagAccess = 0
[ 1122.567347] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0
[ 1122.567351] [001bb829281c2fb8] address between user and kernel address ranges
[ 1122.567356] Internal error: Oops: 0000000096000004 [#1] SMP
[ 1122.567361] Modules linked in: ib_srp scsi_transport_srp target_core_user
[ 1122.575766] ib_srpt:srpt_zerolength_write_done: ib_srpt
10.19.240.81-22 wc->status 0
[ 1122.578490] uio target_core_pscsi target_core_file ib_srpt
target_core_iblock target_core_mod rdma_cm iw_cm ib_cm scsi_debug
rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel null_blk dm_service_time
ib_umad crc32_generic ib_core rfkill sunrpc vfat fat dm_multipath
cavium_rng_vf thunderx_edac cavium_rng ipmi_ssif ipmi_devintf
ipmi_msghandler drm fuse xfs libcrc32c nicvf cavium_ptp crct10dif_ce
ghash_ce sha2_ce sha256_arm64 sha1_ce nicpf thunder_bgx i2c_thunderx
thunder_xcv mdio_thunder mdio_cavium sg dm_mirror dm_region_hash
dm_log dm_mod [last unloaded: null_blk]
[ 1122.694324] CPU: 6 PID: 216 Comm: kworker/6:1 Kdump: loaded Not
tainted 6.6.0-rc4+ #3
[ 1122.709550] Workqueue: ib_cm cm_work_handler [ib_cm]
[ 1122.714576] pstate: 60000005 (nZCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 1122.721537] pc : __kmem_cache_alloc_node+0x100/0x2c0
[ 1122.726506] lr : __kmem_cache_alloc_node+0xb4/0x2c0
[ 1122.731382] sp : ffff800087bcfa10
[ 1122.734692] x29: ffff800087bcfa10 x28: 0000000000000000 x27: b71eb439291cd087
[ 1122.741835] x26: ffff80007cd82774 x25: 00000000ffffffff x24: 0000000000000068
[ 1122.748977] x23: ffff8000820ba000 x22: f71bb829281c2f78 x21: 0000000000000000
[ 1122.756119] x20: 0000000000000cc0 x19: ffff000100010700 x18: ffffffffffffffff
[ 1122.763261] x17: 7a20646575657571 x16: 203a30322d31382e x15: ffff8000826feeff
[ 1122.770403] x14: 0000000000000001 x13: 6174616420657461 x12: 6964656d6d692067
[ 1122.777545] x11: 00000000ffff7fff x10: 00000000ffff7fff x9 : ffff800080398a00
[ 1122.784688] x8 : ffff000148a5c000 x7 : 0000000000000000 x6 : 000000000000007f
[ 1122.791829] x5 : ffff000107bac8c0 x4 : ffff80007cd82774 x3 : 000000000010f006
[ 1122.798971] x2 : b82f1c2829b81bf7 x1 : f71bb829281c2f78 x0 : 0000000000000040
[ 1122.806113] Call trace:
[ 1122.808555] __kmem_cache_alloc_node+0x100/0x2c0
[ 1122.813178] kmalloc_trace+0x40/0x110
[ 1122.816842] srp_alloc_iu.constprop.0+0x3c/0x168 [ib_srp]
[ 1122.822268] srp_alloc_iu_bufs+0xa0/0x240 [ib_srp]
[ 1122.827072] srp_cm_rep_handler+0x2a4/0x2d8 [ib_srp]
[ 1122.832049] srp_rdma_cm_handler+0x104/0x2c0 [ib_srp]
[ 1122.837113] cma_cm_event_handler+0x34/0x140 [rdma_cm]
[ 1122.842306] cma_ib_handler+0x98/0x310 [rdma_cm]
[ 1122.846955] cm_process_work+0x2c/0x1e8 [ib_cm]
[ 1122.851517] cm_queue_work_unlock+0x50/0x138 [ib_cm]
[ 1122.856514] cm_rep_handler+0x250/0x550 [ib_cm]
[ 1122.861075] cm_work_handler+0x2d8/0x3b8 [ib_cm]
[ 1122.865724] process_one_work+0x174/0x3c8
[ 1122.869734] worker_thread+0x2c8/0x3e0
[ 1122.873481] kthread+0x100/0x110
[ 1122.876707] ret_from_fork+0x10/0x20
[ 1122.880286] Code: aa1603e1 f9405e7b 8b0002c2 dac00c42 (f8606ac4)
[ 1122.886391] SMP: stopping secondary CPUs
[ 1122.891420] Starting crashdump kernel...
[ 1122.895341] Bye!
