On Wed, Aug 14, 2013 at 11:50:14AM +0200, Ingo Molnar wrote:
> >
> > Well, I'll have to check if it really doesn't generate additional
> > instructions in generated code, since it's hotpath. I'll ping back once
> > things are done.
>
> An __always_inline should never do that.
Here is the final one, please ping me if something looks not as clean
as it wanted to be and i'll tune code up, thanks!
To hpa@: I had to use explicit @mask because it allows to pass -1ul
mask which is optimized off then by a compiler.
---
From: Cyrill Gorcunov <gorcunov@xxxxxxxxx>
Subject: [PATCH -mm] mm: Unify pte_to_pgoff and pgoff_to_pte helpers
Use unified pte_bfop helper to manipulate bits in pte/pgoff bitfield,
and convert pte_to_pgoff/pgoff_to_pte to inlines.
Signed-off-by: Cyrill Gorcunov <gorcunov@xxxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Andy Lutomirski <luto@xxxxxxxxxxxxxx>
Cc: "H. Peter Anvin" <hpa@xxxxxxxxx>
Cc: Pavel Emelyanov <xemul@xxxxxxxxxxxxx>
Cc: Matt Mackall <mpm@xxxxxxxxxxx>
Cc: Xiao Guangrong <xiaoguangrong@xxxxxxxxxxxxxxxxxx>
Cc: Marcelo Tosatti <mtosatti@xxxxxxxxxx>
Cc: KOSAKI Motohiro <kosaki.motohiro@xxxxxxxxx>
Cc: Stephen Rothwell <sfr@xxxxxxxxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxxxxxxx>
---
arch/x86/include/asm/pgtable-2level.h | 70 +++++++++++++++++++++-------------
1 file changed, 44 insertions(+), 26 deletions(-)
Index: linux-2.6.git/arch/x86/include/asm/pgtable-2level.h
===================================================================
--- linux-2.6.git.orig/arch/x86/include/asm/pgtable-2level.h
+++ linux-2.6.git/arch/x86/include/asm/pgtable-2level.h
@@ -55,8 +55,12 @@ static inline pmd_t native_pmdp_get_and_
#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
#endif
-#define _mfrob(v,r,m,l) ((((v) >> (r)) & (m)) << (l))
-#define __frob(v,r,l) (((v) >> (r)) << (l))
+/* Bit manipulation helper on pte/pgoff entry */
+static inline unsigned long pte_bfop(unsigned long value, unsigned int rightshift,
+ unsigned long mask, unsigned int leftshift)
+{
+ return ((value >> rightshift) & mask) << leftshift;
+}
#ifdef CONFIG_MEM_SOFT_DIRTY
@@ -82,19 +86,26 @@ static inline pmd_t native_pmdp_get_and_
#define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2)
#define PTE_FILE_LSHIFT4 (PTE_FILE_BITS1 + PTE_FILE_BITS2 + PTE_FILE_BITS3)
-#define pte_to_pgoff(pte) \
- (_mfrob((pte).pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + \
- _mfrob((pte).pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + \
- _mfrob((pte).pte_low, PTE_FILE_SHIFT3, PTE_FILE_MASK3, PTE_FILE_LSHIFT3) + \
- __frob((pte).pte_low, PTE_FILE_SHIFT4, PTE_FILE_LSHIFT4))
-
-#define pgoff_to_pte(off) \
- ((pte_t) { .pte_low = \
- _mfrob(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + \
- _mfrob(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + \
- _mfrob(off, PTE_FILE_LSHIFT3, PTE_FILE_MASK3, PTE_FILE_SHIFT3) + \
- __frob(off, PTE_FILE_LSHIFT4, PTE_FILE_SHIFT4) + \
- _PAGE_FILE })
+static __always_inline pgoff_t pte_to_pgoff(pte_t pte)
+{
+ return (pgoff_t)
+ (pte_bfop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) +
+ pte_bfop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) +
+ pte_bfop(pte.pte_low, PTE_FILE_SHIFT3, PTE_FILE_MASK3, PTE_FILE_LSHIFT3) +
+ pte_bfop(pte.pte_low, PTE_FILE_SHIFT4, -1UL, PTE_FILE_LSHIFT4));
+}
+
+static __always_inline pte_t pgoff_to_pte(pgoff_t off)
+{
+ return (pte_t){
+ .pte_low =
+ pte_bfop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) +
+ pte_bfop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) +
+ pte_bfop(off, PTE_FILE_LSHIFT3, PTE_FILE_MASK3, PTE_FILE_SHIFT3) +
+ pte_bfop(off, PTE_FILE_LSHIFT4, -1UL, PTE_FILE_SHIFT4) +
+ _PAGE_FILE,
+ };
+}
#else /* CONFIG_MEM_SOFT_DIRTY */
@@ -120,17 +131,24 @@ static inline pmd_t native_pmdp_get_and_
#define PTE_FILE_LSHIFT2 (PTE_FILE_BITS1)
#define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2)
-#define pte_to_pgoff(pte) \
- (_mfrob((pte).pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + \
- _mfrob((pte).pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + \
- __frob((pte).pte_low, PTE_FILE_SHIFT3, PTE_FILE_LSHIFT3))
-
-#define pgoff_to_pte(off) \
- ((pte_t) { .pte_low = \
- _mfrob(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + \
- _mfrob(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + \
- __frob(off, PTE_FILE_LSHIFT3, PTE_FILE_SHIFT3) + \
- _PAGE_FILE })
+static __always_inline pgoff_t pte_to_pgoff(pte_t pte)
+{
+ return (pgoff_t)
+ (pte_bfop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) +
+ pte_bfop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) +
+ pte_bfop(pte.pte_low, PTE_FILE_SHIFT3, -1UL, PTE_FILE_LSHIFT3));
+}
+
+static __always_inline pte_t pgoff_to_pte(pgoff_t off)
+{
+ return (pte_t){
+ .pte_low =
+ pte_bfop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) +
+ pte_bfop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) +
+ pte_bfop(off, PTE_FILE_LSHIFT3, -1UL, PTE_FILE_SHIFT3) +
+ _PAGE_FILE,
+ };
+}
#endif /* CONFIG_MEM_SOFT_DIRTY */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>