Since 2022/02, the main reference design for the Icicle Kit uses a
non-coherent memory configuration for PCIe, and a shim in the FPGA
fabric, in response to customer requests. As a result, the PCIe root
port has not been usable in mainline since that point in time.
The memory apertures on the reference design have been configured so
that the base of each aperture is mapped to 0x0 in physical memory, a
completely "overlaid" approach. This enables the PCIe root port to
operate in a non-coherent manner (the CPUs/harts use the cached variant
of a particular actual DDR address, the PCIe root port uses the
non-cached variant of the same address. As an example, to access DDR
location 0, the FIC, to which the PCIe rootport is attached, would use
14'0000'0000 and the CPUs would use 10'0000'0000. In this example, the
FIC is responsible for the upper 32-bits of the AXI address and the
root-port (as it is limited to 32-bits on the AXI-S interface) is
responsible for the lower 32-bits of the AXI address. The FPGA designs
utilizing this approach use a simple "shim" to statically set the upper
32-bits of all AXI-S addresses to '000'0014'.
Describe some regions of non-cached memory (and immediately reserve them,
as PolarFire SoC does not support atomics in non-cached memory) so that
we can configure dma-ranges and dma-pools to support a "shim" that will
shift addresses in AXI transactions from the PCIe root port into the
0x14_0000_0000 non-cached region of memory.
The price paid for this is a reduction in the overall system memory.
Signed-off-by: Conor Dooley <conor.dooley@xxxxxxxxxxxxx>
---
.../dts/microchip/mpfs-icicle-kit-fabric.dtsi | 75 ++++++++++++-------
.../boot/dts/microchip/mpfs-icicle-kit.dts | 44 +++++++++--
2 files changed, 84 insertions(+), 35 deletions(-)
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
index 1069134f2e12a..33e76db965bbc 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
@@ -26,33 +26,54 @@ i2c2: i2c@40000200 {
status = "disabled";
};
- pcie: pcie@3000000000 {
- compatible = "microchip,pcie-host-1.0";
- #address-cells = <0x3>;
- #interrupt-cells = <0x1>;
- #size-cells = <0x2>;
- device_type = "pci";
- reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
- reg-names = "cfg", "apb";
- bus-range = <0x0 0x7f>;
- interrupt-parent = <&plic>;
- interrupts = <119>;
- interrupt-map = <0 0 0 1 &pcie_intc 0>,
- <0 0 0 2 &pcie_intc 1>,
- <0 0 0 3 &pcie_intc 2>,
- <0 0 0 4 &pcie_intc 3>;
- interrupt-map-mask = <0 0 0 7>;
- clocks = <&ccc_nw CLK_CCC_PLL0_OUT1>, <&ccc_nw CLK_CCC_PLL0_OUT3>;
- clock-names = "fic1", "fic3";
- ranges = <0x3000000 0x0 0x8000000 0x30 0x8000000 0x0 0x80000000>;
- dma-ranges = <0x02000000 0x0 0x00000000 0x0 0x00000000 0x1 0x00000000>;
- msi-parent = <&pcie>;
- msi-controller;
- status = "disabled";
- pcie_intc: interrupt-controller {
- #address-cells = <0>;
- #interrupt-cells = <1>;
- interrupt-controller;
+ fabric-pcie-bus@3000000000 {
+ compatible = "simple-bus";
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <0x0 0x40000000 0x0 0x40000000 0x0 0x20000000>,
+ <0x30 0x0 0x30 0x0 0x10 0x0>;
+ dma-ranges = <0x0 0x0 0x0 0x80000000 0x0 0x6000000>,
+ <0x0 0x6000000 0x0 0xc6000000 0x0 0x4000000>,
+ <0x0 0xa000000 0x0 0x8a000000 0x0 0x8000000>,
+ <0x0 0x12000000 0x14 0x12000000 0x0 0x10000000>,
+ <0x0 0x22000000 0x10 0x22000000 0x0 0x5e000000>;
+
+ pcie: pcie@3000000000 {
+ compatible = "microchip,pcie-host-1.0";
+ #address-cells = <0x3>;
+ #interrupt-cells = <0x1>;
+ #size-cells = <0x2>;
+ device_type = "pci";
+ dma-noncoherent;
+ reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
+ reg-names = "cfg", "apb";
+ bus-range = <0x0 0x7f>;
+ interrupt-parent = <&plic>;
+ interrupts = <119>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+ <0 0 0 3 &pcie_intc 2>,
+ <0 0 0 4 &pcie_intc 3>;
+ interrupt-map-mask = <0 0 0 7>;
+ clocks = <&ccc_nw CLK_CCC_PLL0_OUT1>, <&ccc_nw CLK_CCC_PLL0_OUT3>;
+ clock-names = "fic1", "fic3";
+ ranges = <0x43000000 0x0 0x9000000 0x30 0x9000000 0x0 0xf000000>,
+ <0x1000000 0x0 0x8000000 0x30 0x8000000 0x0 0x1000000>,
+ <0x3000000 0x0 0x18000000 0x30 0x18000000 0x0 0x70000000>;
+ dma-ranges = <0x3000000 0x0 0x80000000 0x0 0x0 0x0 0x6000000>,
+ <0x3000000 0x0 0x86000000 0x0 0x6000000 0x0 0x4000000>,
+ <0x3000000 0x0 0x8a000000 0x0 0xa000000 0x0 0x8000000>,
+ <0x3000000 0x0 0x92000000 0x0 0x12000000 0x0 0x10000000>,
+ <0x3000000 0x0 0xa2000000 0x0 0x22000000 0x0 0x5e000000>;
+ msi-parent = <&pcie>;
+ msi-controller;
+ status = "disabled";
+
+ pcie_intc: interrupt-controller {
+ #address-cells = <0>;
+ #interrupt-cells = <1>;
+ interrupt-controller;
+ };
};
};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
index f80df225f72b4..6c9340992e9eb 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
@@ -54,16 +54,29 @@ led-4 {
};
};
- ddrc_cache_lo: memory@80000000 {
+ memory@80000000 {
device_type = "memory";
- reg = <0x0 0x80000000 0x0 0x40000000>;
- status = "okay";
+ reg = <0x0 0x80000000 0x0 0x6000000>;
};
- ddrc_cache_hi: memory@1040000000 {
+ memory@8a000000 {
device_type = "memory";
- reg = <0x10 0x40000000 0x0 0x40000000>;
- status = "okay";
+ reg = <0x0 0x8a000000 0x0 0x8000000>;
+ };
+
+ memory@c4000000 {
+ device_type = "memory";
+ reg = <0x0 0xc6000000 0x0 0x4000000>;
+ };
+
+ memory@1022000000 {
+ device_type = "memory";
+ reg = <0x10 0x22000000 0x0 0x5e000000>;
+ };
+
+ memory@1412000000 {
+ device_type = "memory";
+ reg = <0x14 0x12000000 0x0 0x10000000>;
};
reserved-memory {
@@ -71,10 +84,25 @@ reserved-memory {
#size-cells = <2>;
ranges;
- hss_payload: region@BFC00000 {
- reg = <0x0 0xBFC00000 0x0 0x400000>;
+ hss_payload: region@103fc00000 {
+ reg = <0x10 0x3fc00000 0x0 0x400000>;
no-map;
};
+
+ non-cached-low-buffer {
+ compatible = "shared-dma-pool";
+ size = <0x0 0x4000000>;
+ no-map;
+ alloc-ranges = <0x0 0xc6000000 0x0 0x4000000>;
+ };
+
+ non-cached-high-buffer {
+ compatible = "shared-dma-pool";
+ size = <0x0 0x10000000>;
+ no-map;
+ linux,dma-default;
+ alloc-ranges = <0x14 0x12000000 0x0 0x10000000>;
+ };
};
};
--
2.43.2
