Now that we shrunk struct file to 192 bytes aka 3 cachelines reorder
struct file to not leave any holes or have members cross cachelines.
Add a short comment to each of the fields and mark the cachelines.
It's possible that we may have to tweak this based on profiling in the
future. So far I had Jens test this comparing io_uring with non-fixed
and fixed files and it improved performance. The layout is a combination
of Jens' and my changes.
Signed-off-by: Christian Brauner <brauner@xxxxxxxxxx>
---
include/linux/fs.h | 90 +++++++++++++++++++++++++---------------------
1 file changed, 50 insertions(+), 40 deletions(-)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 6c19f87ea615..ace14421d7dc 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -986,52 +986,62 @@ static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index)
index < ra->start + ra->size);
}
-/*
- * f_{lock,count,pos_lock} members can be highly contended and share
- * the same cacheline. f_{lock,mode} are very frequently used together
- * and so share the same cacheline as well. The read-mostly
- * f_{path,inode,op} are kept on a separate cacheline.
+/**
+ * struct file - Represents a file
+ * @f_lock: Protects f_ep, f_flags. Must not be taken from IRQ context.
+ * @f_mode: FMODE_* flags often used in hotpaths
+ * @f_mapping: Contents of a cacheable, mappable object.
+ * @f_flags: file flags
+ * @f_iocb_flags: iocb flags
+ * @private_data: filesystem or driver specific data
+ * @f_path: path of the file
+ * @f_inode: cached inode
+ * @f_count: reference count
+ * @f_pos_lock: lock protecting file position
+ * @f_pos: file position
+ * @f_version: file version
+ * @f_security: LSM security context of this file
+ * @f_owner: file owner
+ * @f_cred: stashed credentials of creator/opener
+ * @f_wb_err: writeback error
+ * @f_sb_err: per sb writeback errors
+ * @f_ep: link of all epoll hooks for this file
+ * @f_task_work: task work entry point
+ * @f_llist: work queue entrypoint
+ * @f_ra: file's readahead state
*/
struct file {
- union {
- /* fput() uses task work when closing and freeing file (default). */
- struct callback_head f_task_work;
- /* fput() must use workqueue (most kernel threads). */
- struct llist_node f_llist;
- /* Invalid after last fput(). */
- struct file_ra_state f_ra;
- };
- /*
- * Protects f_ep, f_flags.
- * Must not be taken from IRQ context.
- */
- spinlock_t f_lock;
- fmode_t f_mode;
- atomic_long_t f_count;
- struct mutex f_pos_lock;
- loff_t f_pos;
- unsigned int f_flags;
- unsigned int f_iocb_flags;
- struct fown_struct *f_owner;
- const struct cred *f_cred;
- struct path f_path;
- struct inode *f_inode; /* cached value */
+ spinlock_t f_lock;
+ fmode_t f_mode;
const struct file_operations *f_op;
-
- u64 f_version;
+ struct address_space *f_mapping;
+ unsigned int f_flags;
+ unsigned int f_iocb_flags;
+ void *private_data;
+ struct path f_path;
+ struct inode *f_inode;
+ /* --- cacheline 1 boundary (64 bytes) --- */
+ atomic_long_t f_count;
+ struct mutex f_pos_lock;
+ loff_t f_pos;
+ u64 f_version;
#ifdef CONFIG_SECURITY
- void *f_security;
+ void *f_security;
#endif
- /* needed for tty driver, and maybe others */
- void *private_data;
-
+ /* --- cacheline 2 boundary (128 bytes) --- */
+ struct fown_struct *f_owner;
+ const struct cred *f_cred;
+ errseq_t f_wb_err;
+ errseq_t f_sb_err;
#ifdef CONFIG_EPOLL
- /* Used by fs/eventpoll.c to link all the hooks to this file */
- struct hlist_head *f_ep;
-#endif /* #ifdef CONFIG_EPOLL */
- struct address_space *f_mapping;
- errseq_t f_wb_err;
- errseq_t f_sb_err; /* for syncfs */
+ struct hlist_head *f_ep;
+#endif
+ union {
+ struct callback_head f_task_work;
+ struct llist_node f_llist;
+ struct file_ra_state f_ra;
+ };
+ /* --- cacheline 2 boundary (192 bytes) --- */
} __randomize_layout
__attribute__((aligned(4))); /* lest something weird decides that 2 is OK */
--
2.43.0
--tqv6c5l7nrp7v7pf--
