Add hardware memory driver, part 2.
The main purpose of hwmem is:
* To allocate buffers suitable for use with hardware. Currently
this means contiguous buffers.
* To synchronize the caches for the allocated buffers. This is
achieved by keeping track of when the CPU uses a buffer and when
other hardware uses the buffer, when we switch from CPU to other
hardware or vice versa the caches are synchronized.
* To handle sharing of allocated buffers between processes i.e.
import, export.
Hwmem is available both through a user space API and through a
kernel API.
Signed-off-by: Johan Mossberg <johan.xx.mossberg@xxxxxxxxxxxxxx>
Acked-by: Linus Walleij <linus.walleij@xxxxxxxxxxxxxx>
---
drivers/misc/hwmem/cache_handler.c | 494 ++++++++++++++++++++++++++++++
drivers/misc/hwmem/cache_handler.h | 60 ++++
drivers/misc/hwmem/cache_handler_u8500.c | 208 +++++++++++++
3 files changed, 762 insertions(+), 0 deletions(-)
create mode 100644 drivers/misc/hwmem/cache_handler.c
create mode 100644 drivers/misc/hwmem/cache_handler.h
create mode 100644 drivers/misc/hwmem/cache_handler_u8500.c
diff --git a/drivers/misc/hwmem/cache_handler.c b/drivers/misc/hwmem/cache_handler.c
new file mode 100644
index 0000000..831770d
--- /dev/null
+++ b/drivers/misc/hwmem/cache_handler.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright (C) ST-Ericsson AB 2010
+ *
+ * Cache handler
+ *
+ * Author: Johan Mossberg <johan.xx.mossberg@xxxxxxxxxxxxxx>
+ * for ST-Ericsson.
+ *
+ * License terms: GNU General Public License (GPL), version 2.
+ */
+
+#include <linux/hwmem.h>
+
+#include <asm/pgtable.h>
+
+#include "cache_handler.h"
+
+#define U32_MAX (~(u32)0)
+
+void cachi_set_buf_cache_settings(struct cach_buf *buf,
+ enum hwmem_alloc_flags cache_settings);
+void cachi_set_pgprot_cache_options(struct cach_buf *buf,
+ pgprot_t *pgprot);
+void cachi_drain_cpu_write_buf(void);
+void cachi_invalidate_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *flushed_everything);
+void cachi_clean_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *cleaned_everything);
+void cachi_flush_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *flushed_everything);
+bool cachi_can_keep_track_of_range_in_cpu_cache(void);
+/* Returns 1 if no cache is present */
+u32 cachi_get_cache_granularity(void);
+
+static void sync_buf_pre_cpu(struct cach_buf *buf, enum hwmem_access access,
+ struct hwmem_region *region);
+static void sync_buf_post_cpu(struct cach_buf *buf,
+ enum hwmem_access next_access, struct hwmem_region *next_region);
+
+static void invalidate_cpu_cache(struct cach_buf *buf,
+ struct cach_range *range_2b_used);
+static void clean_cpu_cache(struct cach_buf *buf,
+ struct cach_range *range_2b_used);
+static void flush_cpu_cache(struct cach_buf *buf,
+ struct cach_range *range_2b_used);
+
+static void null_range(struct cach_range *range);
+static void expand_range(struct cach_range *range,
+ struct cach_range *range_2_add);
+/*
+ * Expands range to one of enclosing_range's two edges. The function will
+ * choose which of enclosing_range's edges to expand range to in such a
+ * way that the size of range is minimized. range must be located inside
+ * enclosing_range.
+ */
+static void expand_range_2_edge(struct cach_range *range,
+ struct cach_range *enclosing_range);
+static void shrink_range(struct cach_range *range,
+ struct cach_range *range_2_remove);
+static bool is_non_empty_range(struct cach_range *range);
+static void intersect_range(struct cach_range *range_1,
+ struct cach_range *range_2, struct cach_range *intersection);
+/* Align_up restrictions apply here to */
+static void align_range_up(struct cach_range *range, u32 alignment);
+static void region_2_range(struct hwmem_region *region, u32 buffer_size,
+ struct cach_range *range);
+
+static u32 offset_2_vaddr(struct cach_buf *buf, u32 offset);
+static u32 offset_2_paddr(struct cach_buf *buf, u32 offset);
+
+/* Saturates, might return unaligned values when that happens */
+static u32 align_up(u32 value, u32 alignment);
+static u32 align_down(u32 value, u32 alignment);
+
+static bool is_wb(enum hwmem_alloc_flags cache_settings);
+static bool is_inner_only(enum hwmem_alloc_flags cache_settings);
+
+/*
+ * Exported functions
+ */
+
+void cach_init_buf(struct cach_buf *buf, enum hwmem_alloc_flags cache_settings,
+ u32 vstart, u32 pstart, u32 size)
+{
+ bool tmp;
+
+ buf->vstart = vstart;
+ buf->pstart = pstart;
+ buf->size = size;
+
+ cachi_set_buf_cache_settings(buf, cache_settings);
+
+ cachi_flush_cpu_cache(offset_2_vaddr(buf, 0),
+ offset_2_vaddr(buf, buf->size), offset_2_paddr(buf, 0),
+ offset_2_paddr(buf, buf->size), false, &tmp);
+ cachi_drain_cpu_write_buf();
+
+ buf->in_cpu_write_buf = false;
+ if (cachi_can_keep_track_of_range_in_cpu_cache())
+ null_range(&buf->range_in_cpu_cache);
+ else {
+ /* Assume worst case, that the entire alloc is in the cache. */
+ buf->range_in_cpu_cache.start = 0;
+ buf->range_in_cpu_cache.end = buf->size;
+ align_range_up(&buf->range_in_cpu_cache,
+ cachi_get_cache_granularity());
+ }
+ null_range(&buf->range_dirty_in_cpu_cache);
+ null_range(&buf->range_invalid_in_cpu_cache);
+}
+
+void cach_set_pgprot_cache_options(struct cach_buf *buf, pgprot_t *pgprot)
+{
+ cachi_set_pgprot_cache_options(buf, pgprot);
+}
+
+void cach_set_domain(struct cach_buf *buf, enum hwmem_access access,
+ enum hwmem_domain domain, struct hwmem_region *region)
+{
+ struct hwmem_region *__region;
+ struct hwmem_region full_region;
+
+ if (region != NULL)
+ __region = region;
+ else {
+ full_region.offset = 0;
+ full_region.count = 1;
+ full_region.start = 0;
+ full_region.end = buf->size;
+ full_region.size = buf->size;
+
+ __region = &full_region;
+ }
+
+ switch (domain) {
+ case HWMEM_DOMAIN_SYNC:
+ sync_buf_post_cpu(buf, access, __region);
+
+ break;
+
+ case HWMEM_DOMAIN_CPU:
+ sync_buf_pre_cpu(buf, access, __region);
+
+ break;
+ }
+}
+
+/*
+ * Local functions
+ */
+
+void __attribute__((weak)) cachi_set_buf_cache_settings(struct cach_buf *buf,
+ enum hwmem_alloc_flags cache_settings)
+{
+ buf->cache_settings = cache_settings & ~HWMEM_ALLOC_CACHE_HINT_MASK;
+
+ if ((cache_settings & HWMEM_ALLOC_CACHED) == HWMEM_ALLOC_CACHED) {
+ /*
+ * If the alloc is cached we'll use the default setting. We
+ * don't know what this setting is so we have to assume the
+ * worst case, ie write back inner and outer.
+ */
+ buf->cache_settings |= HWMEM_ALLOC_CACHE_HINT_WB;
+ }
+}
+
+void __attribute__((weak)) cachi_set_pgprot_cache_options(struct cach_buf *buf,
+ pgprot_t *pgprot)
+{
+ if ((buf->cache_settings & HWMEM_ALLOC_CACHED) == HWMEM_ALLOC_CACHED)
+ *pgprot = *pgprot; /* To silence compiler and checkpatch */
+ else if (buf->cache_settings & HWMEM_ALLOC_BUFFERED)
+ *pgprot = pgprot_writecombine(*pgprot);
+ else
+ *pgprot = pgprot_noncached(*pgprot);
+}
+
+bool __attribute__((weak)) cachi_can_keep_track_of_range_in_cpu_cache(void)
+{
+ /* We don't know so we go with the safe alternative */
+ return false;
+}
+
+static void sync_buf_pre_cpu(struct cach_buf *buf, enum hwmem_access access,
+ struct hwmem_region *region)
+{
+ bool write = access & HWMEM_ACCESS_WRITE;
+ bool read = access & HWMEM_ACCESS_READ;
+
+ if (!write && !read)
+ return;
+
+ if ((buf->cache_settings & HWMEM_ALLOC_CACHED) == HWMEM_ALLOC_CACHED) {
+ struct cach_range region_range;
+
+ region_2_range(region, buf->size, ®ion_range);
+
+ if (read || (write && is_wb(buf->cache_settings)))
+ /* Perform defered invalidates */
+ invalidate_cpu_cache(buf, ®ion_range);
+ if (read)
+ expand_range(&buf->range_in_cpu_cache, ®ion_range);
+ if (write && is_wb(buf->cache_settings)) {
+ struct cach_range intersection;
+
+ intersect_range(&buf->range_in_cpu_cache,
+ ®ion_range, &intersection);
+
+ expand_range(&buf->range_dirty_in_cpu_cache,
+ &intersection);
+ }
+ }
+ if (buf->cache_settings & HWMEM_ALLOC_BUFFERED) {
+ if (write)
+ buf->in_cpu_write_buf = true;
+ }
+}
+
+static void sync_buf_post_cpu(struct cach_buf *buf,
+ enum hwmem_access next_access, struct hwmem_region *next_region)
+{
+ bool write = next_access & HWMEM_ACCESS_WRITE;
+ bool read = next_access & HWMEM_ACCESS_READ;
+ struct cach_range region_range;
+
+ if (!write && !read)
+ return;
+
+ region_2_range(next_region, buf->size, ®ion_range);
+
+ if (write) {
+ if (cachi_can_keep_track_of_range_in_cpu_cache())
+ flush_cpu_cache(buf, ®ion_range);
+ else { /* Defer invalidate */
+ struct cach_range intersection;
+
+ intersect_range(&buf->range_in_cpu_cache,
+ ®ion_range, &intersection);
+
+ expand_range(&buf->range_invalid_in_cpu_cache,
+ &intersection);
+
+ clean_cpu_cache(buf, ®ion_range);
+ }
+ }
+ if (read)
+ clean_cpu_cache(buf, ®ion_range);
+
+ if (buf->in_cpu_write_buf) {
+ cachi_drain_cpu_write_buf();
+
+ buf->in_cpu_write_buf = false;
+ }
+}
+
+static void invalidate_cpu_cache(struct cach_buf *buf, struct cach_range *range)
+{
+ struct cach_range intersection;
+
+ intersect_range(&buf->range_invalid_in_cpu_cache, range,
+ &intersection);
+ if (is_non_empty_range(&intersection)) {
+ bool flushed_everything;
+
+ expand_range_2_edge(&intersection,
+ &buf->range_invalid_in_cpu_cache);
+
+ cachi_invalidate_cpu_cache(
+ offset_2_vaddr(buf, intersection.start),
+ offset_2_vaddr(buf, intersection.end),
+ offset_2_paddr(buf, intersection.start),
+ offset_2_paddr(buf, intersection.end),
+ is_inner_only(buf->cache_settings),
+ &flushed_everything);
+
+ if (flushed_everything) {
+ null_range(&buf->range_invalid_in_cpu_cache);
+ null_range(&buf->range_dirty_in_cpu_cache);
+ } else
+ /*
+ * No need to shrink range_in_cpu_cache as invalidate
+ * is only used when we can't keep track of what's in
+ * the CPU cache.
+ */
+ shrink_range(&buf->range_invalid_in_cpu_cache,
+ &intersection);
+ }
+}
+
+static void clean_cpu_cache(struct cach_buf *buf, struct cach_range *range)
+{
+ struct cach_range intersection;
+
+ intersect_range(&buf->range_dirty_in_cpu_cache, range, &intersection);
+ if (is_non_empty_range(&intersection)) {
+ bool cleaned_everything;
+
+ expand_range_2_edge(&intersection,
+ &buf->range_dirty_in_cpu_cache);
+
+ cachi_clean_cpu_cache(
+ offset_2_vaddr(buf, intersection.start),
+ offset_2_vaddr(buf, intersection.end),
+ offset_2_paddr(buf, intersection.start),
+ offset_2_paddr(buf, intersection.end),
+ is_inner_only(buf->cache_settings),
+ &cleaned_everything);
+
+ if (cleaned_everything)
+ null_range(&buf->range_dirty_in_cpu_cache);
+ else
+ shrink_range(&buf->range_dirty_in_cpu_cache,
+ &intersection);
+ }
+}
+
+static void flush_cpu_cache(struct cach_buf *buf, struct cach_range *range)
+{
+ struct cach_range intersection;
+
+ intersect_range(&buf->range_in_cpu_cache, range, &intersection);
+ if (is_non_empty_range(&intersection)) {
+ bool flushed_everything;
+
+ expand_range_2_edge(&intersection, &buf->range_in_cpu_cache);
+
+ cachi_flush_cpu_cache(
+ offset_2_vaddr(buf, intersection.start),
+ offset_2_vaddr(buf, intersection.end),
+ offset_2_paddr(buf, intersection.start),
+ offset_2_paddr(buf, intersection.end),
+ is_inner_only(buf->cache_settings),
+ &flushed_everything);
+
+ if (flushed_everything) {
+ if (cachi_can_keep_track_of_range_in_cpu_cache())
+ null_range(&buf->range_in_cpu_cache);
+ null_range(&buf->range_dirty_in_cpu_cache);
+ null_range(&buf->range_invalid_in_cpu_cache);
+ } else {
+ if (cachi_can_keep_track_of_range_in_cpu_cache())
+ shrink_range(&buf->range_in_cpu_cache,
+ &intersection);
+ shrink_range(&buf->range_dirty_in_cpu_cache,
+ &intersection);
+ shrink_range(&buf->range_invalid_in_cpu_cache,
+ &intersection);
+ }
+ }
+}
+
+static void null_range(struct cach_range *range)
+{
+ range->start = U32_MAX;
+ range->end = 0;
+}
+
+static void expand_range(struct cach_range *range,
+ struct cach_range *range_2_add)
+{
+ range->start = min(range->start, range_2_add->start);
+ range->end = max(range->end, range_2_add->end);
+}
+
+/*
+ * Expands range to one of enclosing_range's two edges. The function will
+ * choose which of enclosing_range's edges to expand range to in such a
+ * way that the size of range is minimized. range must be located inside
+ * enclosing_range.
+ */
+static void expand_range_2_edge(struct cach_range *range,
+ struct cach_range *enclosing_range)
+{
+ u32 space_on_low_side = range->start - enclosing_range->start;
+ u32 space_on_high_side = enclosing_range->end - range->end;
+
+ if (space_on_low_side < space_on_high_side)
+ range->start = enclosing_range->start;
+ else
+ range->end = enclosing_range->end;
+}
+
+static void shrink_range(struct cach_range *range,
+ struct cach_range *range_2_remove)
+{
+ if (range_2_remove->start > range->start)
+ range->end = min(range->end, range_2_remove->start);
+ else
+ range->start = max(range->start, range_2_remove->end);
+
+ if (range->start >= range->end)
+ null_range(range);
+}
+
+static bool is_non_empty_range(struct cach_range *range)
+{
+ return range->end > range->start;
+}
+
+static void intersect_range(struct cach_range *range_1,
+ struct cach_range *range_2, struct cach_range *intersection)
+{
+ intersection->start = max(range_1->start, range_2->start);
+ intersection->end = min(range_1->end, range_2->end);
+
+ if (intersection->start >= intersection->end)
+ null_range(intersection);
+}
+
+/* Align_up restrictions apply here to */
+static void align_range_up(struct cach_range *range, u32 alignment)
+{
+ if (!is_non_empty_range(range))
+ return;
+
+ range->start = align_down(range->start, alignment);
+ range->end = align_up(range->end, alignment);
+}
+
+static void region_2_range(struct hwmem_region *region, u32 buffer_size,
+ struct cach_range *range)
+{
+ /*
+ * We don't care about invalid regions, instead we limit the region's
+ * range to the buffer's range. This should work good enough, worst
+ * case we synch the entire buffer when we get an invalid region which
+ * is acceptable.
+ */
+ range->start = region->offset + region->start;
+ range->end = min(region->offset + (region->count * region->size) -
+ (region->size - region->end), buffer_size);
+ if (range->start >= range->end) {
+ null_range(range);
+ return;
+ }
+
+ align_range_up(range, cachi_get_cache_granularity());
+}
+
+static u32 offset_2_vaddr(struct cach_buf *buf, u32 offset)
+{
+ return buf->vstart + offset;
+}
+
+static u32 offset_2_paddr(struct cach_buf *buf, u32 offset)
+{
+ return buf->pstart + offset;
+}
+
+/* Saturates, might return unaligned values when that happens */
+static u32 align_up(u32 value, u32 alignment)
+{
+ u32 remainder = value % alignment;
+ u32 value_2_add;
+
+ if (remainder == 0)
+ return value;
+
+ value_2_add = alignment - remainder;
+
+ if (value_2_add > U32_MAX - value) /* Will overflow */
+ return U32_MAX;
+
+ return value + value_2_add;
+}
+
+static u32 align_down(u32 value, u32 alignment)
+{
+ u32 remainder = value % alignment;
+ if (remainder == 0)
+ return value;
+
+ return value - remainder;
+}
+
+static bool is_wb(enum hwmem_alloc_flags cache_settings)
+{
+ u32 cache_hints = cache_settings & HWMEM_ALLOC_CACHE_HINT_MASK;
+ if (cache_hints == HWMEM_ALLOC_CACHE_HINT_WB ||
+ cache_hints == HWMEM_ALLOC_CACHE_HINT_WB_INNER)
+ return true;
+ else
+ return false;
+}
+
+static bool is_inner_only(enum hwmem_alloc_flags cache_settings)
+{
+ u32 cache_hints = cache_settings & HWMEM_ALLOC_CACHE_HINT_MASK;
+ if (cache_hints == HWMEM_ALLOC_CACHE_HINT_WT_INNER ||
+ cache_hints == HWMEM_ALLOC_CACHE_HINT_WB_INNER)
+ return true;
+ else
+ return false;
+}
diff --git a/drivers/misc/hwmem/cache_handler.h b/drivers/misc/hwmem/cache_handler.h
new file mode 100644
index 0000000..3c2a71f
--- /dev/null
+++ b/drivers/misc/hwmem/cache_handler.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) ST-Ericsson AB 2010
+ *
+ * Cache handler
+ *
+ * Author: Johan Mossberg <johan.xx.mossberg@xxxxxxxxxxxxxx>
+ * for ST-Ericsson.
+ *
+ * License terms: GNU General Public License (GPL), version 2.
+ */
+
+/*
+ * Cache handler can not handle simultaneous execution! The caller has to
+ * ensure such a situation does not occur.
+ */
+
+#ifndef _CACHE_HANDLER_H_
+#define _CACHE_HANDLER_H_
+
+#include <linux/types.h>
+#include <linux/hwmem.h>
+
+/*
+ * To not have to double all datatypes we've used hwmem datatypes. If someone
+ * want's to use cache handler but not hwmem then we'll have to define our own
+ * datatypes.
+ */
+
+struct cach_range {
+ u32 start; /* Inclusive */
+ u32 end; /* Exclusive */
+};
+
+/*
+ * Internal, do not touch!
+ */
+struct cach_buf {
+ u32 vstart;
+ u32 pstart;
+ u32 size;
+
+ /* Remaining hints are active */
+ enum hwmem_alloc_flags cache_settings;
+
+ bool in_cpu_write_buf;
+ struct cach_range range_in_cpu_cache;
+ struct cach_range range_dirty_in_cpu_cache;
+ struct cach_range range_invalid_in_cpu_cache;
+};
+
+void cach_init_buf(struct cach_buf *buf,
+ enum hwmem_alloc_flags cache_settings, u32 vstart, u32 pstart,
+ u32 size);
+
+void cach_set_pgprot_cache_options(struct cach_buf *buf, pgprot_t *pgprot);
+
+void cach_set_domain(struct cach_buf *buf, enum hwmem_access access,
+ enum hwmem_domain domain, struct hwmem_region *region);
+
+#endif /* _CACHE_HANDLER_H_ */
diff --git a/drivers/misc/hwmem/cache_handler_u8500.c b/drivers/misc/hwmem/cache_handler_u8500.c
new file mode 100644
index 0000000..3c1bc5a
--- /dev/null
+++ b/drivers/misc/hwmem/cache_handler_u8500.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) ST-Ericsson AB 2010
+ *
+ * Cache handler
+ *
+ * Author: Johan Mossberg <johan.xx.mossberg@xxxxxxxxxxxxxx>
+ * for ST-Ericsson.
+ *
+ * License terms: GNU General Public License (GPL), version 2.
+ */
+
+/* TODO: Move all this stuff to mach */
+
+#include <linux/hwmem.h>
+#include <linux/dma-mapping.h>
+
+#include <asm/pgtable.h>
+#include <asm/cacheflush.h>
+#include <asm/outercache.h>
+#include <asm/system.h>
+
+#include "cache_handler.h"
+
+/*
+ * Values are derived from measurements on HREFP_1.1_V32_OM_S10 running
+ * u8500-android-2.2_r1.1_v0.21.
+ *
+ * A lot of time can be spent trying to figure out the perfect breakpoints but
+ * for now I've chosen the following simple way.
+ *
+ * breakpoint = best_case + (worst_case - best_case) * 0.666
+ * The breakpoint is moved slightly towards the worst case because a full
+ * clean/flush affects the entire system so we should be a bit careful.
+ *
+ * BEST CASE:
+ * Best case is that the cache is empty and the system is idling. The case
+ * where the cache contains only targeted data could be better in some cases
+ * but it's hard to do measurements and calculate on that case so I choose the
+ * easier alternative.
+ *
+ * inner_inv_breakpoint = time_2_range_inv_on_empty_cache(
+ * complete_flush_on_empty_cache_time)
+ * inner_clean_breakpoint = time_2_range_clean_on_empty_cache(
+ * complete_clean_on_empty_cache_time)
+ *
+ * outer_inv_breakpoint = time_2_range_inv_on_empty_cache(
+ * complete_flush_on_empty_cache_time)
+ * outer_clean_breakpoint = time_2_range_clean_on_empty_cache(
+ * complete_clean_on_empty_cache_time)
+ * outer_flush_breakpoint = time_2_range_flush_on_empty_cache(
+ * complete_flush_on_empty_cache_time)
+ *
+ * WORST CASE:
+ * Worst case is that the cache is filled with dirty non targeted data that
+ * will be used after the synchronization and the system is under heavy load.
+ *
+ * inner_inv_breakpoint = time_2_range_inv_on_empty_cache(
+ * complete_flush_on_full_cache_time * 1.5 +
+ * complete_flush_on_full_cache_time / 2)
+ * Times 1.5 because it runs on both cores half the time. Plus
+ * "complete_flush_on_full_cache_time / 2" because all data has to be read
+ * back, here we assume that both cores can fill their cache simultaneously
+ * (seems to be the case as operations on full and empty inner cache takes
+ * roughlythe same amount of time ie the bus to outer is not the bottle neck).
+ * inner_clean_breakpoint = time_2_range_clean_on_empty_cache(
+ * complete_clean_on_full_cache_time * 1.5)
+ *
+ * outer_inv_breakpoint = time_2_range_inv_on_empty_cache(
+ * complete_flush_on_full_cache_time * 2 +
+ * (complete_flush_on_full_cache_time -
+ * complete_flush_on_empty_cache_time) * 2)
+ * Plus "(complete_flush_on_full_cache_time -
+ * complete_flush_on_empty_cache_time)" because no one else can work when we
+ * hog the bus with our unecessary transfer.
+ * outer_clean_breakpoint = time_2_range_clean_on_empty_cache(
+ * complete_clean_on_full_cache_time +
+ * (complete_clean_on_full_cache_time -
+ * complete_clean_on_empty_cache_time))
+ * outer_flush_breakpoint = time_2_range_flush_on_empty_cache(
+ * complete_flush_on_full_cache_time * 2 +
+ * (complete_flush_on_full_cache_time -
+ * complete_flush_on_empty_cache_time) * 2)
+ *
+ * These values might have to be updated if changes are made to the CPU, L2$,
+ * memory bus or memory.
+ */
+/* 36224 */
+static const u32 inner_inv_breakpoint = 21324 + (43697 - 21324) * 0.666;
+/* 28930 */
+static const u32 inner_clean_breakpoint = 21324 + (32744 - 21324) * 0.666;
+/* 485414 */
+static const u32 outer_inv_breakpoint = 68041 + (694727 - 68041) * 0.666;
+/* 254069 */
+static const u32 outer_clean_breakpoint = 68041 + (347363 - 68041) * 0.666;
+/* 485414 */
+static const u32 outer_flush_breakpoint = 68041 + (694727 - 68041) * 0.666;
+
+static bool is_wt(enum hwmem_alloc_flags cache_settings);
+
+void cachi_set_buf_cache_settings(struct cach_buf *buf,
+ enum hwmem_alloc_flags cache_settings)
+{
+ buf->cache_settings = cache_settings & ~HWMEM_ALLOC_CACHE_HINT_MASK;
+
+ if ((cache_settings & HWMEM_ALLOC_CACHED) == HWMEM_ALLOC_CACHED) {
+ if (is_wt(cache_settings))
+ buf->cache_settings |= HWMEM_ALLOC_CACHE_HINT_WT;
+ else
+ buf->cache_settings |= HWMEM_ALLOC_CACHE_HINT_WB;
+ }
+}
+
+void cachi_set_pgprot_cache_options(struct cach_buf *buf, pgprot_t *pgprot)
+{
+ if ((buf->cache_settings & HWMEM_ALLOC_CACHED) == HWMEM_ALLOC_CACHED) {
+ if (is_wt(buf->cache_settings))
+ *pgprot = __pgprot_modify(*pgprot, L_PTE_MT_MASK,
+ L_PTE_MT_WRITETHROUGH);
+ else
+ *pgprot = __pgprot_modify(*pgprot, L_PTE_MT_MASK,
+ L_PTE_MT_WRITEBACK);
+ } else if (buf->cache_settings & HWMEM_ALLOC_BUFFERED)
+ *pgprot = pgprot_writecombine(*pgprot);
+ else
+ *pgprot = pgprot_noncached(*pgprot);
+}
+
+void cachi_drain_cpu_write_buf(void)
+{
+ dsb();
+ outer_cache.sync();
+}
+
+void cachi_invalidate_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *flushed_everything)
+{
+ u32 range_size = virt_end - virt_start;
+
+ *flushed_everything = false;
+
+ if (range_size < outer_inv_breakpoint)
+ outer_cache.inv_range(phys_start, phys_end);
+ else
+ outer_cache.flush_all();
+
+ /* Inner invalidate range */
+ dmac_map_area((void *)virt_start, range_size, DMA_FROM_DEVICE);
+}
+
+void cachi_clean_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *cleaned_everything)
+{
+ u32 range_size = virt_end - virt_start;
+
+ *cleaned_everything = false;
+
+ /* Inner clean range */
+ dmac_map_area((void *)virt_start, range_size, DMA_TO_DEVICE);
+
+ /*
+ * There is currently no outer_cache.clean_all() so we use flush
+ * instead, which is ok as clean is a subset of flush. Clean range
+ * and flush range take the same amount of time so we can use
+ * outer_flush_breakpoint here.
+ */
+ if (range_size < outer_flush_breakpoint)
+ outer_cache.clean_range(phys_start, phys_end);
+ else
+ outer_cache.flush_all();
+}
+
+void cachi_flush_cpu_cache(u32 virt_start, u32 virt_end, u32 phys_start,
+ u32 phys_end, bool inner_only, bool *flushed_everything)
+{
+ u32 range_size = virt_end - virt_start;
+
+ *flushed_everything = false;
+
+ /* Inner clean range */
+ dmac_map_area((void *)virt_start, range_size, DMA_TO_DEVICE);
+
+ if (range_size < outer_flush_breakpoint)
+ outer_cache.flush_range(phys_start, phys_end);
+ else
+ outer_cache.flush_all();
+
+ /* Inner invalidate range */
+ dmac_map_area((void *)virt_start, range_size, DMA_FROM_DEVICE);
+}
+
+u32 cachi_get_cache_granularity(void)
+{
+ return 32;
+}
+
+/*
+ * Local functions
+ */
+
+static bool is_wt(enum hwmem_alloc_flags cache_settings)
+{
+ u32 cache_hints = cache_settings & HWMEM_ALLOC_CACHE_HINT_MASK;
+ if (cache_hints == HWMEM_ALLOC_CACHE_HINT_WT ||
+ cache_hints == HWMEM_ALLOC_CACHE_HINT_WT_INNER)
+ return true;
+ else
+ return false;
+}
--
1.6.3.3
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxxx For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom policy in Canada: sign http://dissolvethecrtc.ca/
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>