Hi Prahakar-san,
> From: Lad, Prabhakar, Sent: Wednesday, April 1, 2020 6:24 PM
<snip>
> > The crc32 value was calculated before print the buffer.
> > This means that timing is difference between crc32_le(addr) and print_buffer(addr).
> > So, I'd like to copy the addr buffer to tmp and use the tmp for crc32_le() and print_buffer().
> >
> Following is the diff of suggested changes:
Thanks!
> diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
> index 3c49514b4813..b782134838d1 100644
> --- a/drivers/misc/pci_endpoint_test.c
> +++ b/drivers/misc/pci_endpoint_test.c
> @@ -561,6 +561,23 @@ static bool pci_endpoint_test_write(struct
> pci_endpoint_test *test,
> return ret;
> }
>
> +static void print_buffer(struct device *dev, void *buff_addr, size_t size)
> +{
> + size_t i;
> + u64 *addr = buff_addr;
> +
> + for(i = 0; i < size; i += sizeof(addr))
> + dev_err(dev, "buf[%zu] : %llx\n", i, *addr);
> +
> + for(i = 0; i < size; i +=1) {
> + if (0x910c690d == crc32_le(~0, buff_addr, i))
> + dev_err(dev, "%x\n",
> + crc32_le(~0, buff_addr, i));
> + }
> +
> + dev_err(dev, "\n\n\n\n");
> +}
> +
> static bool pci_endpoint_test_read(struct pci_endpoint_test *test,
> unsigned long arg)
> {
> @@ -574,6 +591,7 @@ static bool pci_endpoint_test_read(struct
> pci_endpoint_test *test,
> struct pci_dev *pdev = test->pdev;
> struct device *dev = &pdev->dev;
> void *orig_addr;
> + void *tmp_buffer;
> dma_addr_t orig_phys_addr;
> size_t offset;
> size_t alignment = test->alignment;
> @@ -601,14 +619,19 @@ static bool pci_endpoint_test_read(struct
> pci_endpoint_test *test,
> }
>
> orig_addr = kzalloc(size + alignment, GFP_KERNEL);
> - if (!orig_addr) {
> + tmp_buffer = kzalloc(size + alignment, GFP_KERNEL);
> + if (!orig_addr || !tmp_buffer) {
> dev_err(dev, "Failed to allocate destination address\n");
> ret = false;
> + if (orig_addr)
> + kfree(orig_addr);
> + if (tmp_buffer)
> + kfree(tmp_buffer);
> goto err;
> }
>
> orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
> - DMA_FROM_DEVICE);
> + DMA_BIDIRECTIONAL);
> if (dma_mapping_error(dev, orig_phys_addr)) {
> dev_err(dev, "failed to map source buffer address\n");
> ret = false;
> @@ -640,14 +663,24 @@ static bool pci_endpoint_test_read(struct
> pci_endpoint_test *test,
> wait_for_completion(&test->irq_raised);
>
> dma_unmap_single(dev, orig_phys_addr, size + alignment,
> - DMA_FROM_DEVICE);
> + DMA_BIDIRECTIONAL);
> +
> + memcpy(tmp_buffer, addr, size);
>
> crc32 = crc32_le(~0, addr, size);
I intended to use tmp_buffer for this crc32_le().
But, the result seems my guess (timing issue) is incorrect anyway.
> if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
> ret = true;
>
> + print_buffer(dev, addr, size);
> + print_buffer(dev, tmp_buffer, size);
> + dev_err(dev, "%s kzalloc:%px dma handle:%llx buffer CRC:%x BAR
> CRC:%x tmp buffer crc: %x\n",
> + __func__, orig_addr, orig_phys_addr, crc32,
> + pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM),
> + crc32_le(~0, tmp_buffer, size));
Calling crc32_le(~0, tmp_buffer, size) here is useful.
> +
> err_phys_addr:
> kfree(orig_addr);
> + kfree(tmp_buffer);
> err:
> return ret;
> }
<snip>
> > By the way, your PCIe host environment (RZ/G2N) output the following log?
> >
> > [ 0.000000] software IO TLB: mapped [mem 0x7bfff000-0x7ffff000] (64MB)
> >
> > If so, I guess this issue is related to this software IO TLB behavior.
> > IIUC, if we use coherent buffer or GPF_DMA buffer, the kernel will not
> > use software IO TLB for these buffers.
> >
>
> root@hihope-rzg2m:~# dmesg | grep TLB
> [ 0.000000] software IO TLB: mapped [mem 0x7bfff000-0x7ffff000] (64MB)
I got it.
<snip>
> [ 88.963900] pci-endpoint-test 0000:01:00.0: pci_endpoint_test_read
> kzalloc:ffff0004b6393000 dma handle:7e93c800 buffer CRC:ce535039 BAR
> CRC:910c690d tmp buffer crc: ce535039
> READ ( 2176 bytes): NOT OKAY
>
> And starangely even tmp_buffer has the value of 0xDF, but the CRC is screwed.
I agree. this result is strange.
So, I have no idea why this issue happened..
By the way, according to previous your report, you are using pci/next branch
and the branch is based on v5.6-rc1. There is no evidence though,
I'd like to use next-20200401 tag from linux-next repo to use v5.6 based kernel
whether the strange issue happens on the latest v5.6 kernel code or not.
Note that I confirmed the next-20200401 tag has a commit d293237739414d
("misc: pci_endpoint_test: Use streaming DMA APIs for buffer allocation").
Best regards,
Yoshihiro Shimoda
