This patch is to enable the 64-bit swiotlb buffer.
The state of the art swiotlb pre-allocates <=32-bit memory in order to meet
the DMA mask requirement for some 32-bit legacy device. Considering most
devices nowadays support 64-bit DMA and IOMMU is available, the swiotlb is
not used for most of the times, except:
1. The xen PVM domain requires the DMA addresses to both (1) less than the
dma mask, and (2) continuous in machine address. Therefore, the 64-bit
device may still require swiotlb on PVM domain.
2. From source code the AMD SME/SEV will enable SWIOTLB_FORCE. As a result
it is always required to allocate from swiotlb buffer even the device dma
mask is 64-bit.
sme_early_init()
-> if (sev_active())
swiotlb_force = SWIOTLB_FORCE;
Therefore, this patch introduces the 2nd swiotlb buffer for 64-bit DMA
access. For instance, the swiotlb_tbl_map_single() allocates from the 2nd
64-bit buffer if the device DMA mask is
min_not_zero(*hwdev->dma_mask, hwdev->bus_dma_limit).
The example to configure 64-bit swiotlb is "swiotlb=65536,524288,force"
or "swiotlb=,524288,force", where 524288 is the size of 64-bit buffer.
With the patch, the kernel will be able to allocate >4GB swiotlb buffer.
This patch is only for swiotlb, not including xen-swiotlb.
Cc: Joe Jin <joe.jin@xxxxxxxxxx>
Signed-off-by: Dongli Zhang <dongli.zhang@xxxxxxxxxx>
---
arch/mips/cavium-octeon/dma-octeon.c | 3 +-
arch/powerpc/kernel/dma-swiotlb.c | 2 +-
arch/powerpc/platforms/pseries/svm.c | 2 +-
arch/x86/kernel/pci-swiotlb.c | 5 +-
arch/x86/pci/sta2x11-fixup.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_internal.c | 4 +-
drivers/gpu/drm/i915/i915_scatterlist.h | 2 +-
drivers/gpu/drm/nouveau/nouveau_ttm.c | 2 +-
drivers/mmc/host/sdhci.c | 2 +-
drivers/pci/xen-pcifront.c | 2 +-
drivers/xen/swiotlb-xen.c | 9 +-
include/linux/swiotlb.h | 28 +-
kernel/dma/swiotlb.c | 339 +++++++++++--------
13 files changed, 238 insertions(+), 164 deletions(-)
diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c
index df70308db0e6..3480555d908a 100644
--- a/arch/mips/cavium-octeon/dma-octeon.c
+++ b/arch/mips/cavium-octeon/dma-octeon.c
@@ -245,6 +245,7 @@ void __init plat_swiotlb_setup(void)
panic("%s: Failed to allocate %zu bytes align=%lx\n",
__func__, swiotlbsize, PAGE_SIZE);
- if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs, 1) == -ENOMEM)
+ if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs,
+ SWIOTLB_LO, 1) == -ENOMEM)
panic("Cannot allocate SWIOTLB buffer");
}
diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
index fc7816126a40..88113318c53f 100644
--- a/arch/powerpc/kernel/dma-swiotlb.c
+++ b/arch/powerpc/kernel/dma-swiotlb.c
@@ -20,7 +20,7 @@ void __init swiotlb_detect_4g(void)
static int __init check_swiotlb_enabled(void)
{
if (ppc_swiotlb_enable)
- swiotlb_print_info();
+ swiotlb_print_info(SWIOTLB_LO);
else
swiotlb_exit();
diff --git a/arch/powerpc/platforms/pseries/svm.c b/arch/powerpc/platforms/pseries/svm.c
index 9f8842d0da1f..77910e4ffad8 100644
--- a/arch/powerpc/platforms/pseries/svm.c
+++ b/arch/powerpc/platforms/pseries/svm.c
@@ -52,7 +52,7 @@ void __init svm_swiotlb_init(void)
bytes = io_tlb_nslabs << IO_TLB_SHIFT;
vstart = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE);
- if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, false))
+ if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, SWIOTLB_LO, false))
return;
if (io_tlb_start[SWIOTLB_LO])
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index c2cfa5e7c152..950624fd95a4 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -67,12 +67,15 @@ void __init pci_swiotlb_init(void)
void __init pci_swiotlb_late_init(void)
{
+ int i;
+
/* An IOMMU turned us off. */
if (!swiotlb)
swiotlb_exit();
else {
printk(KERN_INFO "PCI-DMA: "
"Using software bounce buffering for IO (SWIOTLB)\n");
- swiotlb_print_info();
+ for (i = 0; i < swiotlb_nr; i++)
+ swiotlb_print_info(i);
}
}
diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c
index 7d2525691854..c440520b2055 100644
--- a/arch/x86/pci/sta2x11-fixup.c
+++ b/arch/x86/pci/sta2x11-fixup.c
@@ -57,7 +57,7 @@ static void sta2x11_new_instance(struct pci_dev *pdev)
int size = STA2X11_SWIOTLB_SIZE;
/* First instance: register your own swiotlb area */
dev_info(&pdev->dev, "Using SWIOTLB (size %i)\n", size);
- if (swiotlb_late_init_with_default_size(size))
+ if (swiotlb_late_init_with_default_size(size), SWIOTLB_LO)
dev_emerg(&pdev->dev, "init swiotlb failed\n");
}
list_add(&instance->list, &sta2x11_instance_list);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c
index ad22f42541bd..947683f2e568 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c
@@ -42,10 +42,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
max_order = MAX_ORDER;
#ifdef CONFIG_SWIOTLB
- if (swiotlb_nr_tbl()) {
+ if (swiotlb_nr_tbl(SWIOTLB_LO)) {
unsigned int max_segment;
- max_segment = swiotlb_max_segment();
+ max_segment = swiotlb_max_segment(SWIOTLB_LO);
if (max_segment) {
max_segment = max_t(unsigned int, max_segment,
PAGE_SIZE) >> PAGE_SHIFT;
diff --git a/drivers/gpu/drm/i915/i915_scatterlist.h b/drivers/gpu/drm/i915/i915_scatterlist.h
index 9cb26a224034..c63c7f6941f6 100644
--- a/drivers/gpu/drm/i915/i915_scatterlist.h
+++ b/drivers/gpu/drm/i915/i915_scatterlist.h
@@ -118,7 +118,7 @@ static inline unsigned int i915_sg_page_sizes(struct scatterlist *sg)
static inline unsigned int i915_sg_segment_size(void)
{
- unsigned int size = swiotlb_max_segment();
+ unsigned int size = swiotlb_max_segment(SWIOTLB_LO);
if (size == 0)
size = UINT_MAX;
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index a37bc3d7b38b..0919b207ac47 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -321,7 +321,7 @@ nouveau_ttm_init(struct nouveau_drm *drm)
}
#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
- need_swiotlb = !!swiotlb_nr_tbl();
+ need_swiotlb = !!swiotlb_nr_tbl(SWIOTLB_LO);
#endif
ret = ttm_bo_device_init(&drm->ttm.bdev, &nouveau_bo_driver,
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 646823ddd317..1f7fb912d5a9 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -4582,7 +4582,7 @@ int sdhci_setup_host(struct sdhci_host *host)
mmc->max_segs = SDHCI_MAX_SEGS;
} else if (host->flags & SDHCI_USE_SDMA) {
mmc->max_segs = 1;
- if (swiotlb_max_segment()) {
+ if (swiotlb_max_segment(SWIOTLB_LO)) {
unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
IO_TLB_SEGSIZE;
mmc->max_req_size = min(mmc->max_req_size,
diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c
index c6fe0cfec0f6..9509ed9b4126 100644
--- a/drivers/pci/xen-pcifront.c
+++ b/drivers/pci/xen-pcifront.c
@@ -693,7 +693,7 @@ static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
spin_unlock(&pcifront_dev_lock);
- if (!err && !swiotlb_nr_tbl()) {
+ if (!err && !swiotlb_nr_tbl(SWIOTLB_LO)) {
err = pci_xen_swiotlb_init_late();
if (err)
dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 3261880ad859..662638093542 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -184,7 +184,7 @@ int __ref xen_swiotlb_init(int verbose, bool early)
enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
unsigned int repeat = 3;
- xen_io_tlb_nslabs = swiotlb_nr_tbl();
+ xen_io_tlb_nslabs = swiotlb_nr_tbl(SWIOTLB_LO);
retry:
bytes = xen_set_nslabs(xen_io_tlb_nslabs);
order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
@@ -245,16 +245,17 @@ int __ref xen_swiotlb_init(int verbose, bool early)
}
if (early) {
if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs,
- verbose))
+ SWIOTLB_LO, verbose))
panic("Cannot allocate SWIOTLB buffer");
rc = 0;
} else
- rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
+ rc = swiotlb_late_init_with_tbl(xen_io_tlb_start,
+ xen_io_tlb_nslabs, SWIOTLB_LO);
end:
xen_io_tlb_end = xen_io_tlb_start + bytes;
if (!rc)
- swiotlb_set_max_segment(PAGE_SIZE);
+ swiotlb_set_max_segment(PAGE_SIZE, SWIOTLB_LO);
return rc;
error:
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 3d5980d77810..8ba45fddfb14 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -43,11 +43,14 @@ extern int swiotlb_nr;
#define IO_TLB_DEFAULT_SIZE (64UL<<20)
extern void swiotlb_init(int verbose);
-int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
-extern unsigned long swiotlb_nr_tbl(void);
+int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs,
+ enum swiotlb_t type, int verbose);
+extern unsigned long swiotlb_nr_tbl(enum swiotlb_t type);
unsigned long swiotlb_size_or_default(void);
-extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
-extern int swiotlb_late_init_with_default_size(size_t default_size);
+extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs,
+ enum swiotlb_t type);
+extern int swiotlb_late_init_with_default_size(size_t default_size,
+ enum swiotlb_t type);
extern void __init swiotlb_update_mem_attributes(void);
/*
@@ -83,8 +86,13 @@ extern phys_addr_t io_tlb_start[], io_tlb_end[];
static inline bool is_swiotlb_buffer(phys_addr_t paddr)
{
- return paddr >= io_tlb_start[SWIOTLB_LO] &&
- paddr < io_tlb_end[SWIOTLB_LO];
+ int i;
+
+ for (i = 0; i < swiotlb_nr; i++)
+ if (paddr >= io_tlb_start[i] && paddr < io_tlb_end[i])
+ return true;
+
+ return false;
}
static inline int swiotlb_get_type(phys_addr_t paddr)
@@ -99,7 +107,7 @@ static inline int swiotlb_get_type(phys_addr_t paddr)
}
void __init swiotlb_exit(void);
-unsigned int swiotlb_max_segment(void);
+unsigned int swiotlb_max_segment(enum swiotlb_t type);
size_t swiotlb_max_mapping_size(struct device *dev);
bool is_swiotlb_active(void);
void __init swiotlb_adjust_size(unsigned long new_size);
@@ -112,7 +120,7 @@ static inline bool is_swiotlb_buffer(phys_addr_t paddr)
static inline void swiotlb_exit(void)
{
}
-static inline unsigned int swiotlb_max_segment(void)
+static inline unsigned int swiotlb_max_segment(enum swiotlb_t type)
{
return 0;
}
@@ -131,7 +139,7 @@ static inline void swiotlb_adjust_size(unsigned long new_size)
}
#endif /* CONFIG_SWIOTLB */
-extern void swiotlb_print_info(void);
-extern void swiotlb_set_max_segment(unsigned int);
+extern void swiotlb_print_info(enum swiotlb_t type);
+extern void swiotlb_set_max_segment(unsigned int val, enum swiotlb_t type);
#endif /* __LINUX_SWIOTLB_H */
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index c91d3d2c3936..cd28db5b016a 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -111,6 +111,11 @@ static int late_alloc;
int swiotlb_nr = 1;
+static const char * const swiotlb_name[] = {
+ "Low Mem",
+ "High Mem"
+};
+
static int __init
setup_io_tlb_npages(char *str)
{
@@ -121,11 +126,25 @@ setup_io_tlb_npages(char *str)
}
if (*str == ',')
++str;
+
+ if (isdigit(*str)) {
+ io_tlb_nslabs[SWIOTLB_HI] = simple_strtoul(str, &str, 0);
+ /* avoid tail segment of size < IO_TLB_SEGSIZE */
+ io_tlb_nslabs[SWIOTLB_HI] = ALIGN(io_tlb_nslabs[SWIOTLB_HI], IO_TLB_SEGSIZE);
+
+ swiotlb_nr = 2;
+ }
+
+ if (*str == ',')
+ ++str;
+
if (!strcmp(str, "force")) {
swiotlb_force = SWIOTLB_FORCE;
} else if (!strcmp(str, "noforce")) {
swiotlb_force = SWIOTLB_NO_FORCE;
io_tlb_nslabs[SWIOTLB_LO] = 1;
+ if (swiotlb_nr > 1)
+ io_tlb_nslabs[SWIOTLB_HI] = 1;
}
return 0;
@@ -134,24 +153,24 @@ early_param("swiotlb", setup_io_tlb_npages);
static bool no_iotlb_memory[SWIOTLB_MAX];
-unsigned long swiotlb_nr_tbl(void)
+unsigned long swiotlb_nr_tbl(enum swiotlb_t type)
{
- return unlikely(no_iotlb_memory[SWIOTLB_LO]) ? 0 : io_tlb_nslabs[SWIOTLB_LO];
+ return unlikely(no_iotlb_memory[type]) ? 0 : io_tlb_nslabs[type];
}
EXPORT_SYMBOL_GPL(swiotlb_nr_tbl);
-unsigned int swiotlb_max_segment(void)
+unsigned int swiotlb_max_segment(enum swiotlb_t type)
{
- return unlikely(no_iotlb_memory[SWIOTLB_LO]) ? 0 : max_segment[SWIOTLB_LO];
+ return unlikely(no_iotlb_memory[type]) ? 0 : max_segment[type];
}
EXPORT_SYMBOL_GPL(swiotlb_max_segment);
-void swiotlb_set_max_segment(unsigned int val)
+void swiotlb_set_max_segment(unsigned int val, enum swiotlb_t type)
{
if (swiotlb_force == SWIOTLB_FORCE)
- max_segment[SWIOTLB_LO] = 1;
+ max_segment[type] = 1;
else
- max_segment[SWIOTLB_LO] = rounddown(val, PAGE_SIZE);
+ max_segment[type] = rounddown(val, PAGE_SIZE);
}
unsigned long swiotlb_size_or_default(void)
@@ -181,18 +200,18 @@ void __init swiotlb_adjust_size(unsigned long new_size)
}
}
-void swiotlb_print_info(void)
+void swiotlb_print_info(enum swiotlb_t type)
{
- unsigned long bytes = io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT;
+ unsigned long bytes = io_tlb_nslabs[type] << IO_TLB_SHIFT;
- if (no_iotlb_memory[SWIOTLB_LO]) {
- pr_warn("No low mem\n");
+ if (no_iotlb_memory[type]) {
+ pr_warn("No low mem with %s\n", swiotlb_name[type]);
return;
}
- pr_info("mapped [mem %pa-%pa] (%luMB)\n",
- &io_tlb_start[SWIOTLB_LO], &io_tlb_end[SWIOTLB_LO],
- bytes >> 20);
+ pr_info("%s mapped [mem %pa-%pa] (%luMB)\n",
+ swiotlb_name[type],
+ &io_tlb_start[type], &io_tlb_end[type], bytes >> 20);
}
/*
@@ -205,88 +224,104 @@ void __init swiotlb_update_mem_attributes(void)
{
void *vaddr;
unsigned long bytes;
+ int i;
- if (no_iotlb_memory[SWIOTLB_LO] || late_alloc)
- return;
+ for (i = 0; i < swiotlb_nr; i++) {
+ if (no_iotlb_memory[i] || late_alloc)
+ continue;
- vaddr = phys_to_virt(io_tlb_start[SWIOTLB_LO]);
- bytes = PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT);
- set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
- memset(vaddr, 0, bytes);
+ vaddr = phys_to_virt(io_tlb_start[i]);
+ bytes = PAGE_ALIGN(io_tlb_nslabs[i] << IO_TLB_SHIFT);
+ set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
+ memset(vaddr, 0, bytes);
+ }
}
-int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
+int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs,
+ enum swiotlb_t type, int verbose)
{
unsigned long i, bytes;
size_t alloc_size;
bytes = nslabs << IO_TLB_SHIFT;
- io_tlb_nslabs[SWIOTLB_LO] = nslabs;
- io_tlb_start[SWIOTLB_LO] = __pa(tlb);
- io_tlb_end[SWIOTLB_LO] = io_tlb_start[SWIOTLB_LO] + bytes;
+ io_tlb_nslabs[type] = nslabs;
+ io_tlb_start[type] = __pa(tlb);
+ io_tlb_end[type] = io_tlb_start[type] + bytes;
/*
* Allocate and initialize the free list array. This array is used
* to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
* between io_tlb_start and io_tlb_end.
*/
- alloc_size = PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] * sizeof(int));
- io_tlb_list[SWIOTLB_LO] = memblock_alloc(alloc_size, PAGE_SIZE);
- if (!io_tlb_list[SWIOTLB_LO])
+ alloc_size = PAGE_ALIGN(io_tlb_nslabs[type] * sizeof(int));
+ io_tlb_list[type] = memblock_alloc(alloc_size, PAGE_SIZE);
+ if (!io_tlb_list[type])
panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);
- alloc_size = PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] * sizeof(phys_addr_t));
- io_tlb_orig_addr[SWIOTLB_LO] = memblock_alloc(alloc_size, PAGE_SIZE);
- if (!io_tlb_orig_addr[SWIOTLB_LO])
+ alloc_size = PAGE_ALIGN(io_tlb_nslabs[type] * sizeof(phys_addr_t));
+ io_tlb_orig_addr[type] = memblock_alloc(alloc_size, PAGE_SIZE);
+ if (!io_tlb_orig_addr[type])
panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);
- for (i = 0; i < io_tlb_nslabs[SWIOTLB_LO]; i++) {
- io_tlb_list[SWIOTLB_LO][i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
- io_tlb_orig_addr[SWIOTLB_LO][i] = INVALID_PHYS_ADDR;
+ for (i = 0; i < io_tlb_nslabs[type]; i++) {
+ io_tlb_list[type][i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
+ io_tlb_orig_addr[type][i] = INVALID_PHYS_ADDR;
}
- io_tlb_index[SWIOTLB_LO] = 0;
- no_iotlb_memory[SWIOTLB_LO] = false;
+ io_tlb_index[type] = 0;
+ no_iotlb_memory[type] = false;
if (verbose)
- swiotlb_print_info();
+ swiotlb_print_info(type);
- swiotlb_set_max_segment(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT);
+ swiotlb_set_max_segment(io_tlb_nslabs[type] << IO_TLB_SHIFT, type);
return 0;
}
-/*
- * Statically reserve bounce buffer space and initialize bounce buffer data
- * structures for the software IO TLB used to implement the DMA API.
- */
-void __init
-swiotlb_init(int verbose)
+static void __init
+swiotlb_init_type(enum swiotlb_t type, int verbose)
{
size_t default_size = IO_TLB_DEFAULT_SIZE;
unsigned char *vstart;
unsigned long bytes;
- if (!io_tlb_nslabs[SWIOTLB_LO]) {
- io_tlb_nslabs[SWIOTLB_LO] = (default_size >> IO_TLB_SHIFT);
- io_tlb_nslabs[SWIOTLB_LO] = ALIGN(io_tlb_nslabs[SWIOTLB_LO], IO_TLB_SEGSIZE);
+ if (!io_tlb_nslabs[type]) {
+ io_tlb_nslabs[type] = (default_size >> IO_TLB_SHIFT);
+ io_tlb_nslabs[type] = ALIGN(io_tlb_nslabs[type], IO_TLB_SEGSIZE);
}
- bytes = io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT;
+ bytes = io_tlb_nslabs[type] << IO_TLB_SHIFT;
+
+ if (type == SWIOTLB_LO)
+ vstart = memblock_alloc_low(PAGE_ALIGN(bytes), PAGE_SIZE);
+ else
+ vstart = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE);
- /* Get IO TLB memory from the low pages */
- vstart = memblock_alloc_low(PAGE_ALIGN(bytes), PAGE_SIZE);
- if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs[SWIOTLB_LO], verbose))
+ if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs[type], type, verbose))
return;
- if (io_tlb_start[SWIOTLB_LO]) {
- memblock_free_early(io_tlb_start[SWIOTLB_LO],
- PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT));
- io_tlb_start[SWIOTLB_LO] = 0;
+ if (io_tlb_start[type]) {
+ memblock_free_early(io_tlb_start[type],
+ PAGE_ALIGN(io_tlb_nslabs[type] << IO_TLB_SHIFT));
+ io_tlb_start[type] = 0;
}
- pr_warn("Cannot allocate buffer");
- no_iotlb_memory[SWIOTLB_LO] = true;
+ pr_warn("Cannot allocate buffer %s\n", swiotlb_name[type]);
+ no_iotlb_memory[type] = true;
+}
+
+/*
+ * Statically reserve bounce buffer space and initialize bounce buffer data
+ * structures for the software IO TLB used to implement the DMA API.
+ */
+void __init
+swiotlb_init(int verbose)
+{
+ int i;
+
+ for (i = 0; i < swiotlb_nr; i++)
+ swiotlb_init_type(i, verbose);
}
/*
@@ -295,67 +330,68 @@ swiotlb_init(int verbose)
* This should be just like above, but with some error catching.
*/
int
-swiotlb_late_init_with_default_size(size_t default_size)
+swiotlb_late_init_with_default_size(size_t default_size, enum swiotlb_t type)
{
- unsigned long bytes, req_nslabs = io_tlb_nslabs[SWIOTLB_LO];
+ unsigned long bytes, req_nslabs = io_tlb_nslabs[type];
unsigned char *vstart = NULL;
unsigned int order;
int rc = 0;
+ gfp_t gfp_mask = (type == SWIOTLB_LO) ? GFP_DMA | __GFP_NOWARN :
+ GFP_KERNEL | __GFP_NOWARN;
- if (!io_tlb_nslabs[SWIOTLB_LO]) {
- io_tlb_nslabs[SWIOTLB_LO] = (default_size >> IO_TLB_SHIFT);
- io_tlb_nslabs[SWIOTLB_LO] = ALIGN(io_tlb_nslabs[SWIOTLB_LO], IO_TLB_SEGSIZE);
+ if (!io_tlb_nslabs[type]) {
+ io_tlb_nslabs[type] = (default_size >> IO_TLB_SHIFT);
+ io_tlb_nslabs[type] = ALIGN(io_tlb_nslabs[type], IO_TLB_SEGSIZE);
}
/*
* Get IO TLB memory from the low pages
*/
- order = get_order(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT);
- io_tlb_nslabs[SWIOTLB_LO] = SLABS_PER_PAGE << order;
- bytes = io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT;
+ order = get_order(io_tlb_nslabs[type] << IO_TLB_SHIFT);
+ io_tlb_nslabs[type] = SLABS_PER_PAGE << order;
+ bytes = io_tlb_nslabs[type] << IO_TLB_SHIFT;
while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
- vstart = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
- order);
+ vstart = (void *)__get_free_pages(gfp_mask, order);
if (vstart)
break;
order--;
}
if (!vstart) {
- io_tlb_nslabs[SWIOTLB_LO] = req_nslabs;
+ io_tlb_nslabs[type] = req_nslabs;
return -ENOMEM;
}
if (order != get_order(bytes)) {
pr_warn("only able to allocate %ld MB\n",
(PAGE_SIZE << order) >> 20);
- io_tlb_nslabs[SWIOTLB_LO] = SLABS_PER_PAGE << order;
+ io_tlb_nslabs[type] = SLABS_PER_PAGE << order;
}
- rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs[SWIOTLB_LO]);
+ rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs[type], type);
if (rc)
free_pages((unsigned long)vstart, order);
return rc;
}
-static void swiotlb_cleanup(void)
+static void swiotlb_cleanup(enum swiotlb_t type)
{
- io_tlb_end[SWIOTLB_LO] = 0;
- io_tlb_start[SWIOTLB_LO] = 0;
- io_tlb_nslabs[SWIOTLB_LO] = 0;
- max_segment[SWIOTLB_LO] = 0;
+ io_tlb_end[type] = 0;
+ io_tlb_start[type] = 0;
+ io_tlb_nslabs[type] = 0;
+ max_segment[type] = 0;
}
int
-swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
+swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs, enum swiotlb_t type)
{
unsigned long i, bytes;
bytes = nslabs << IO_TLB_SHIFT;
- io_tlb_nslabs[SWIOTLB_LO] = nslabs;
- io_tlb_start[SWIOTLB_LO] = virt_to_phys(tlb);
- io_tlb_end[SWIOTLB_LO] = io_tlb_start[SWIOTLB_LO] + bytes;
+ io_tlb_nslabs[type] = nslabs;
+ io_tlb_start[type] = virt_to_phys(tlb);
+ io_tlb_end[type] = io_tlb_start[type] + bytes;
set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
memset(tlb, 0, bytes);
@@ -365,63 +401,67 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
* to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
* between io_tlb_start and io_tlb_end.
*/
- io_tlb_list[SWIOTLB_LO] = (unsigned int *)__get_free_pages(GFP_KERNEL,
- get_order(io_tlb_nslabs[SWIOTLB_LO] * sizeof(int)));
- if (!io_tlb_list[SWIOTLB_LO])
+ io_tlb_list[type] = (unsigned int *)__get_free_pages(GFP_KERNEL,
+ get_order(io_tlb_nslabs[type] * sizeof(int)));
+ if (!io_tlb_list[type])
goto cleanup3;
- io_tlb_orig_addr[SWIOTLB_LO] = (phys_addr_t *)
+ io_tlb_orig_addr[type] = (phys_addr_t *)
__get_free_pages(GFP_KERNEL,
- get_order(io_tlb_nslabs[SWIOTLB_LO] *
+ get_order(io_tlb_nslabs[type] *
sizeof(phys_addr_t)));
- if (!io_tlb_orig_addr[SWIOTLB_LO])
+ if (!io_tlb_orig_addr[type])
goto cleanup4;
- for (i = 0; i < io_tlb_nslabs[SWIOTLB_LO]; i++) {
- io_tlb_list[SWIOTLB_LO][i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
- io_tlb_orig_addr[SWIOTLB_LO][i] = INVALID_PHYS_ADDR;
+ for (i = 0; i < io_tlb_nslabs[type]; i++) {
+ io_tlb_list[type][i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
+ io_tlb_orig_addr[type][i] = INVALID_PHYS_ADDR;
}
- io_tlb_index[SWIOTLB_LO] = 0;
- no_iotlb_memory[SWIOTLB_LO] = false;
+ io_tlb_index[type] = 0;
+ no_iotlb_memory[type] = false;
- swiotlb_print_info();
+ swiotlb_print_info(type);
late_alloc = 1;
- swiotlb_set_max_segment(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT);
+ swiotlb_set_max_segment(io_tlb_nslabs[type] << IO_TLB_SHIFT, type);
return 0;
cleanup4:
- free_pages((unsigned long)io_tlb_list[SWIOTLB_LO], get_order(io_tlb_nslabs[SWIOTLB_LO] *
+ free_pages((unsigned long)io_tlb_list[type], get_order(io_tlb_nslabs[type] *
sizeof(int)));
- io_tlb_list[SWIOTLB_LO] = NULL;
+ io_tlb_list[type] = NULL;
cleanup3:
- swiotlb_cleanup();
+ swiotlb_cleanup(type);
return -ENOMEM;
}
void __init swiotlb_exit(void)
{
- if (!io_tlb_orig_addr[SWIOTLB_LO])
- return;
+ int i;
- if (late_alloc) {
- free_pages((unsigned long)io_tlb_orig_addr[SWIOTLB_LO],
- get_order(io_tlb_nslabs[SWIOTLB_LO] * sizeof(phys_addr_t)));
- free_pages((unsigned long)io_tlb_list[SWIOTLB_LO],
- get_order(io_tlb_nslabs[SWIOTLB_LO] * sizeof(int)));
- free_pages((unsigned long)phys_to_virt(io_tlb_start[SWIOTLB_LO]),
- get_order(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT));
- } else {
- memblock_free_late(__pa(io_tlb_orig_addr[SWIOTLB_LO]),
- PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] * sizeof(phys_addr_t)));
- memblock_free_late(__pa(io_tlb_list[SWIOTLB_LO]),
- PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] * sizeof(int)));
- memblock_free_late(io_tlb_start[SWIOTLB_LO],
- PAGE_ALIGN(io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT));
+ for (i = 0; i < swiotlb_nr; i++) {
+ if (!io_tlb_orig_addr[i])
+ continue;
+
+ if (late_alloc) {
+ free_pages((unsigned long)io_tlb_orig_addr[i],
+ get_order(io_tlb_nslabs[i] * sizeof(phys_addr_t)));
+ free_pages((unsigned long)io_tlb_list[i],
+ get_order(io_tlb_nslabs[i] * sizeof(int)));
+ free_pages((unsigned long)phys_to_virt(io_tlb_start[i]),
+ get_order(io_tlb_nslabs[i] << IO_TLB_SHIFT));
+ } else {
+ memblock_free_late(__pa(io_tlb_orig_addr[i]),
+ PAGE_ALIGN(io_tlb_nslabs[i] * sizeof(phys_addr_t)));
+ memblock_free_late(__pa(io_tlb_list[i]),
+ PAGE_ALIGN(io_tlb_nslabs[i] * sizeof(int)));
+ memblock_free_late(io_tlb_start[i],
+ PAGE_ALIGN(io_tlb_nslabs[i] << IO_TLB_SHIFT));
+ }
+ swiotlb_cleanup(i);
}
- swiotlb_cleanup();
}
/*
@@ -468,7 +508,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size,
enum dma_data_direction dir, unsigned long attrs)
{
- dma_addr_t tbl_dma_addr = phys_to_dma_unencrypted(hwdev, io_tlb_start[SWIOTLB_LO]);
+ dma_addr_t tbl_dma_addr;
unsigned long flags;
phys_addr_t tlb_addr;
unsigned int nslots, stride, index, wrap;
@@ -477,8 +517,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
unsigned long offset_slots;
unsigned long max_slots;
unsigned long tmp_io_tlb_used;
+ unsigned long dma_mask = min_not_zero(*hwdev->dma_mask,
+ hwdev->bus_dma_limit);
+ int type;
- if (no_iotlb_memory[SWIOTLB_LO])
+ if (swiotlb_nr > 1 && dma_mask == DMA_BIT_MASK(64))
+ type = SWIOTLB_HI;
+ else
+ type = SWIOTLB_LO;
+
+ if (no_iotlb_memory[type])
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
if (mem_encrypt_active())
@@ -490,6 +538,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
return (phys_addr_t)DMA_MAPPING_ERROR;
}
+ tbl_dma_addr = phys_to_dma_unencrypted(hwdev, io_tlb_start[type]);
+
mask = dma_get_seg_boundary(hwdev);
tbl_dma_addr &= mask;
@@ -521,11 +571,11 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
*/
spin_lock_irqsave(&io_tlb_lock, flags);
- if (unlikely(nslots > io_tlb_nslabs[SWIOTLB_LO] - io_tlb_used[SWIOTLB_LO]))
+ if (unlikely(nslots > io_tlb_nslabs[type] - io_tlb_used[type]))
goto not_found;
- index = ALIGN(io_tlb_index[SWIOTLB_LO], stride);
- if (index >= io_tlb_nslabs[SWIOTLB_LO])
+ index = ALIGN(io_tlb_index[type], stride);
+ if (index >= io_tlb_nslabs[type])
index = 0;
wrap = index;
@@ -533,7 +583,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
while (iommu_is_span_boundary(index, nslots, offset_slots,
max_slots)) {
index += stride;
- if (index >= io_tlb_nslabs[SWIOTLB_LO])
+ if (index >= io_tlb_nslabs[type])
index = 0;
if (index == wrap)
goto not_found;
@@ -544,42 +594,42 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
* contiguous buffers, we allocate the buffers from that slot
* and mark the entries as '0' indicating unavailable.
*/
- if (io_tlb_list[SWIOTLB_LO][index] >= nslots) {
+ if (io_tlb_list[type][index] >= nslots) {
int count = 0;
for (i = index; i < (int) (index + nslots); i++)
- io_tlb_list[SWIOTLB_LO][i] = 0;
+ io_tlb_list[type][i] = 0;
for (i = index - 1;
(OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) &&
- io_tlb_list[SWIOTLB_LO][i];
+ io_tlb_list[type][i];
i--)
- io_tlb_list[SWIOTLB_LO][i] = ++count;
- tlb_addr = io_tlb_start[SWIOTLB_LO] + (index << IO_TLB_SHIFT);
+ io_tlb_list[type][i] = ++count;
+ tlb_addr = io_tlb_start[type] + (index << IO_TLB_SHIFT);
/*
* Update the indices to avoid searching in the next
* round.
*/
- io_tlb_index[SWIOTLB_LO] = ((index + nslots) < io_tlb_nslabs[SWIOTLB_LO]
+ io_tlb_index[type] = ((index + nslots) < io_tlb_nslabs[type]
? (index + nslots) : 0);
goto found;
}
index += stride;
- if (index >= io_tlb_nslabs[SWIOTLB_LO])
+ if (index >= io_tlb_nslabs[type])
index = 0;
} while (index != wrap);
not_found:
- tmp_io_tlb_used = io_tlb_used[SWIOTLB_LO];
+ tmp_io_tlb_used = io_tlb_used[type];
spin_unlock_irqrestore(&io_tlb_lock, flags);
if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
- dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
- alloc_size, io_tlb_nslabs[SWIOTLB_LO], tmp_io_tlb_used);
+ dev_warn(hwdev, "%s swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
+ swiotlb_name[type], alloc_size, io_tlb_nslabs[type], tmp_io_tlb_used);
return (phys_addr_t)DMA_MAPPING_ERROR;
found:
- io_tlb_used[SWIOTLB_LO] += nslots;
+ io_tlb_used[type] += nslots;
spin_unlock_irqrestore(&io_tlb_lock, flags);
/*
@@ -588,7 +638,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
* needed.
*/
for (i = 0; i < nslots; i++)
- io_tlb_orig_addr[SWIOTLB_LO][index+i] = orig_addr + (i << IO_TLB_SHIFT);
+ io_tlb_orig_addr[type][index+i] = orig_addr + (i << IO_TLB_SHIFT);
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE);
@@ -605,8 +655,9 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
{
unsigned long flags;
int i, count, nslots = ALIGN(alloc_size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
- int index = (tlb_addr - io_tlb_start[SWIOTLB_LO]) >> IO_TLB_SHIFT;
- phys_addr_t orig_addr = io_tlb_orig_addr[SWIOTLB_LO][index];
+ int type = swiotlb_get_type(tlb_addr);
+ int index = (tlb_addr - io_tlb_start[type]) >> IO_TLB_SHIFT;
+ phys_addr_t orig_addr = io_tlb_orig_addr[type][index];
/*
* First, sync the memory before unmapping the entry
@@ -625,14 +676,14 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
spin_lock_irqsave(&io_tlb_lock, flags);
{
count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
- io_tlb_list[SWIOTLB_LO][index + nslots] : 0);
+ io_tlb_list[type][index + nslots] : 0);
/*
* Step 1: return the slots to the free list, merging the
* slots with superceeding slots
*/
for (i = index + nslots - 1; i >= index; i--) {
- io_tlb_list[SWIOTLB_LO][i] = ++count;
- io_tlb_orig_addr[SWIOTLB_LO][i] = INVALID_PHYS_ADDR;
+ io_tlb_list[type][i] = ++count;
+ io_tlb_orig_addr[type][i] = INVALID_PHYS_ADDR;
}
/*
* Step 2: merge the returned slots with the preceding slots,
@@ -640,11 +691,11 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
*/
for (i = index - 1;
(OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) &&
- io_tlb_list[SWIOTLB_LO][i];
+ io_tlb_list[type][i];
i--)
- io_tlb_list[SWIOTLB_LO][i] = ++count;
+ io_tlb_list[type][i] = ++count;
- io_tlb_used[SWIOTLB_LO] -= nslots;
+ io_tlb_used[type] -= nslots;
}
spin_unlock_irqrestore(&io_tlb_lock, flags);
}
@@ -653,8 +704,9 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir,
enum dma_sync_target target)
{
- int index = (tlb_addr - io_tlb_start[SWIOTLB_LO]) >> IO_TLB_SHIFT;
- phys_addr_t orig_addr = io_tlb_orig_addr[SWIOTLB_LO][index];
+ int type = swiotlb_get_type(tlb_addr);
+ int index = (tlb_addr - io_tlb_start[type]) >> IO_TLB_SHIFT;
+ phys_addr_t orig_addr = io_tlb_orig_addr[type][index];
if (orig_addr == INVALID_PHYS_ADDR)
return;
@@ -737,6 +789,15 @@ static int __init swiotlb_create_debugfs(void)
root = debugfs_create_dir("swiotlb", NULL);
debugfs_create_ulong("io_tlb_nslabs", 0400, root, &io_tlb_nslabs[SWIOTLB_LO]);
debugfs_create_ulong("io_tlb_used", 0400, root, &io_tlb_used[SWIOTLB_LO]);
+
+ if (swiotlb_nr == 1)
+ return 0;
+
+ debugfs_create_ulong("io_tlb_nslabs-highmem", 0400,
+ root, &io_tlb_nslabs[SWIOTLB_HI]);
+ debugfs_create_ulong("io_tlb_used-highmem", 0400,
+ root, &io_tlb_used[SWIOTLB_HI]);
+
return 0;
}
--
2.17.1
