CONFIG_DEBUG_STACK_USAGE is enabled by default on most architectures.
Its purpose is to determine and print the maximum stack depth on
thread exit.
The way it works, is it starts from the buttom of the stack and
searches the first non-zero word in the stack. With dynamic stack it
does not work very well, as it means it faults every pages in every
stack.
Instead, add a specific version of stack_not_used() for dynamic stacks
where instead of starting from the buttom of the stack, we start from
the last page mapped in the stack.
In addition to not doing uncessary page faulting, this search is
optimized by skipping search through zero pages.
Also, because dynamic stack does not end with MAGIC_NUMBER, there is
no need to skeep the buttom most word in the stack.
Signed-off-by: Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
---
arch/Kconfig | 1 -
include/linux/sched/task_stack.h | 38 +++++++++++++++++++++++---------
2 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index da3df347b069..759b2bb7edb6 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1266,7 +1266,6 @@ config DYNAMIC_STACK
depends on HAVE_ARCH_DYNAMIC_STACK
depends on VMAP_STACK
depends on !KASAN
- depends on !DEBUG_STACK_USAGE
depends on !STACK_GROWSUP
help
Dynamic kernel stacks allow to save memory on machines with a lot of
diff --git a/include/linux/sched/task_stack.h b/include/linux/sched/task_stack.h
index 396d5418ae32..c5fb679b31ee 100644
--- a/include/linux/sched/task_stack.h
+++ b/include/linux/sched/task_stack.h
@@ -9,6 +9,7 @@
#include <linux/sched.h>
#include <linux/magic.h>
#include <linux/refcount.h>
+#include <linux/vmalloc.h>
#ifdef CONFIG_THREAD_INFO_IN_TASK
@@ -109,6 +110,21 @@ static inline void dynamic_stack(struct task_struct *tsk)
static inline void set_task_stack_end_magic(struct task_struct *tsk) {}
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long stack_not_used(struct task_struct *p)
+{
+ struct vm_struct *vm_area = p->stack_vm_area;
+ unsigned long alloc_size = vm_area->nr_pages << PAGE_SHIFT;
+ unsigned long stack = (unsigned long)p->stack;
+ unsigned long *n = (unsigned long *)(stack + THREAD_SIZE - alloc_size);
+
+ while (!*n)
+ n++;
+
+ return (unsigned long)n - stack;
+}
+#endif /* CONFIG_DEBUG_STACK_USAGE */
+
#else /* !CONFIG_DYNAMIC_STACK */
#define task_stack_end_corrupted(task) \
@@ -123,17 +139,6 @@ static inline bool dynamic_stack_fault(struct task_struct *tsk,
return false;
}
-#endif /* CONFIG_DYNAMIC_STACK */
-
-static inline int object_is_on_stack(const void *obj)
-{
- void *stack = task_stack_page(current);
-
- return (obj >= stack) && (obj < (stack + THREAD_SIZE));
-}
-
-extern void thread_stack_cache_init(void);
-
#ifdef CONFIG_DEBUG_STACK_USAGE
#ifdef CONFIG_STACK_GROWSUP
static inline unsigned long stack_not_used(struct task_struct *p)
@@ -160,6 +165,17 @@ static inline unsigned long stack_not_used(struct task_struct *p)
#endif /* CONFIG_STACK_GROWSUP */
#endif /* CONFIG_DEBUG_STACK_USAGE */
+#endif /* CONFIG_DYNAMIC_STACK */
+
+static inline int object_is_on_stack(const void *obj)
+{
+ void *stack = task_stack_page(current);
+
+ return (obj >= stack) && (obj < (stack + THREAD_SIZE));
+}
+
+extern void thread_stack_cache_init(void);
+
static inline int kstack_end(void *addr)
{
/* Reliable end of stack detection:
--
2.44.0.278.ge034bb2e1d-goog
