Move (most of) the definitions of hashtab_search(), hashtab_insert(),
and hashtab_map() to the header file and make them inline. In
combination with the previous patch, this avoids calling the callbacks
indirectly by function pointers and allows better optimization, leading
to a drastic performance improvement of these operations.
For example, the duration of security_transition_sid() (which spends a
lot of its time doing hashtab lookups after recnt optimizations) when
called from selinux_msg_queue_msgsnd() is cut by half thanks to these
patches. This was measured by analyzing the following command using the
perf tool:
stress-ng --msg 1 --msg-ops 4000000
Signed-off-by: Ondrej Mosnacek <omosnace@xxxxxxxxxx>
---
security/selinux/ss/hashtab.c | 79 +++-----------------------------
security/selinux/ss/hashtab.h | 84 +++++++++++++++++++++++++++++++----
2 files changed, 81 insertions(+), 82 deletions(-)
diff --git a/security/selinux/ss/hashtab.c b/security/selinux/ss/hashtab.c
index 8126b909a757..e5af05770e43 100644
--- a/security/selinux/ss/hashtab.c
+++ b/security/selinux/ss/hashtab.c
@@ -7,7 +7,6 @@
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/errno.h>
-#include <linux/sched.h>
#include "hashtab.h"
static struct kmem_cache *hashtab_node_cachep;
@@ -40,71 +39,23 @@ int hashtab_init(struct hashtab *h, u32 nel_hint)
return h->htable ? 0 : -ENOMEM;
}
-int hashtab_insert(struct hashtab *h, void *key, void *datum,
- struct hashtab_key_params key_params)
+int __hashtab_insert(struct hashtab *h, struct hashtab_node **dst,
+ void *key, void *datum)
{
- u32 hvalue;
- struct hashtab_node *prev, *cur, *newnode;
-
- cond_resched();
-
- if (!h->size || h->nel == HASHTAB_MAX_NODES)
- return -EINVAL;
-
- hvalue = key_params.hash(key) & (h->size - 1);
- prev = NULL;
- cur = h->htable[hvalue];
- while (cur) {
- int cmp = key_params.cmp(key, cur->key);
-
- if (cmp == 0)
- return -EEXIST;
- if (cmp < 0)
- break;
- prev = cur;
- cur = cur->next;
- }
+ struct hashtab_node *newnode;
newnode = kmem_cache_zalloc(hashtab_node_cachep, GFP_KERNEL);
if (!newnode)
return -ENOMEM;
newnode->key = key;
newnode->datum = datum;
- if (prev) {
- newnode->next = prev->next;
- prev->next = newnode;
- } else {
- newnode->next = h->htable[hvalue];
- h->htable[hvalue] = newnode;
- }
+ newnode->next = *dst;
+ *dst = newnode;
h->nel++;
return 0;
}
-void *hashtab_search(struct hashtab *h, const void *key,
- struct hashtab_key_params key_params)
-{
- u32 hvalue;
- struct hashtab_node *cur;
-
- if (!h->size)
- return NULL;
-
- hvalue = key_params.hash(key) & (h->size - 1);
- cur = h->htable[hvalue];
- while (cur) {
- int cmp = key_params.cmp(key, cur->key);
-
- if (cmp == 0)
- return cur->datum;
- if (cmp < 0)
- break;
- cur = cur->next;
- }
- return NULL;
-}
-
void hashtab_destroy(struct hashtab *h)
{
u32 i;
@@ -124,26 +75,6 @@ void hashtab_destroy(struct hashtab *h)
h->htable = NULL;
}
-int hashtab_map(struct hashtab *h,
- int (*apply)(void *k, void *d, void *args),
- void *args)
-{
- u32 i;
- int ret;
- struct hashtab_node *cur;
-
- for (i = 0; i < h->size; i++) {
- cur = h->htable[i];
- while (cur) {
- ret = apply(cur->key, cur->datum, args);
- if (ret)
- return ret;
- cur = cur->next;
- }
- }
- return 0;
-}
-
void hashtab_stat(struct hashtab *h, struct hashtab_info *info)
{
diff --git a/security/selinux/ss/hashtab.h b/security/selinux/ss/hashtab.h
index 4885234257d4..a6ccf695405b 100644
--- a/security/selinux/ss/hashtab.h
+++ b/security/selinux/ss/hashtab.h
@@ -11,7 +11,10 @@
#ifndef _SS_HASHTAB_H_
#define _SS_HASHTAB_H_
-#define HASHTAB_MAX_NODES 0xffffffff
+#include <linux/errno.h>
+#include <linux/sched.h>
+
+#define HASHTAB_MAX_NODES U32_MAX
struct hashtab_key_params {
u32 (*hash)(const void *key); /* hash function */
@@ -43,6 +46,9 @@ struct hashtab_info {
*/
int hashtab_init(struct hashtab *h, u32 nel_hint);
+int __hashtab_insert(struct hashtab *h, struct hashtab_node **dst,
+ void *key, void *datum);
+
/*
* Inserts the specified (key, datum) pair into the specified hash table.
*
@@ -51,8 +57,34 @@ int hashtab_init(struct hashtab *h, u32 nel_hint);
* -EINVAL for general errors or
0 otherwise.
*/
-int hashtab_insert(struct hashtab *h, void *k, void *d,
- struct hashtab_key_params key_params);
+static inline int hashtab_insert(struct hashtab *h, void *key, void *datum,
+ struct hashtab_key_params key_params)
+{
+ u32 hvalue;
+ struct hashtab_node *prev, *cur;
+
+ cond_resched();
+
+ if (!h->size || h->nel == HASHTAB_MAX_NODES)
+ return -EINVAL;
+
+ hvalue = key_params.hash(key) & (h->size - 1);
+ prev = NULL;
+ cur = h->htable[hvalue];
+ while (cur) {
+ int cmp = key_params.cmp(key, cur->key);
+
+ if (cmp == 0)
+ return -EEXIST;
+ if (cmp < 0)
+ break;
+ prev = cur;
+ cur = cur->next;
+ }
+
+ return __hashtab_insert(h, prev ? &prev->next : &h->htable[hvalue],
+ key, datum);
+}
/*
* Searches for the entry with the specified key in the hash table.
@@ -60,8 +92,28 @@ int hashtab_insert(struct hashtab *h, void *k, void *d,
* Returns NULL if no entry has the specified key or
* the datum of the entry otherwise.
*/
-void *hashtab_search(struct hashtab *h, const void *k,
- struct hashtab_key_params key_params);
+static inline void *hashtab_search(struct hashtab *h, const void *key,
+ struct hashtab_key_params key_params)
+{
+ u32 hvalue;
+ struct hashtab_node *cur;
+
+ if (!h->size)
+ return NULL;
+
+ hvalue = key_params.hash(key) & (h->size - 1);
+ cur = h->htable[hvalue];
+ while (cur) {
+ int cmp = key_params.cmp(key, cur->key);
+
+ if (cmp == 0)
+ return cur->datum;
+ if (cmp < 0)
+ break;
+ cur = cur->next;
+ }
+ return NULL;
+}
/*
* Destroys the specified hash table.
@@ -79,9 +131,25 @@ void hashtab_destroy(struct hashtab *h);
* iterating through the hash table and will propagate the error
* return to its caller.
*/
-int hashtab_map(struct hashtab *h,
- int (*apply)(void *k, void *d, void *args),
- void *args);
+static inline int hashtab_map(struct hashtab *h,
+ int (*apply)(void *k, void *d, void *args),
+ void *args)
+{
+ u32 i;
+ int ret;
+ struct hashtab_node *cur;
+
+ for (i = 0; i < h->size; i++) {
+ cur = h->htable[i];
+ while (cur) {
+ ret = apply(cur->key, cur->datum, args);
+ if (ret)
+ return ret;
+ cur = cur->next;
+ }
+ }
+ return 0;
+}
/* Fill info with some hash table statistics */
void hashtab_stat(struct hashtab *h, struct hashtab_info *info);
--
2.25.4
