Allow nfsdctl to use the new POOL_MODE_GET/SET operations to configure
the pool-mode on the host. This setting mirrors the behavior of the
sunrpc module parameter. The patch also adds support for a new pool-mode
option in /etc/nfs.conf. With this change, you can now configure a
multipool NFS server by just properly setting up /etc/nfs.conf.
Signed-off-by: Jeff Layton <jlayton@xxxxxxxxxx>
---
configure.ac | 2 +-
utils/nfsdctl/nfsd_netlink.h | 10 +++
utils/nfsdctl/nfsdctl.8 | 23 ++++++-
utils/nfsdctl/nfsdctl.adoc | 12 ++++
utils/nfsdctl/nfsdctl.c | 154 ++++++++++++++++++++++++++++++++++++++-----
5 files changed, 183 insertions(+), 18 deletions(-)
diff --git a/configure.ac b/configure.ac
index 8cb42ebc5846..1aebc3a767a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -265,7 +265,7 @@ AC_ARG_ENABLE(nfsdctl,
# ensure we have the pool-mode commands
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <linux/nfsd_netlink.h>]],
- [[int foo = NFSD_CMD_LISTENER_GET;]])],
+ [[int foo = NFSD_CMD_POOL_MODE_GET;]])],
[AC_DEFINE([USE_SYSTEM_NFSD_NETLINK_H], 1,
["Use system's linux/nfsd_netlink.h"])])
fi
diff --git a/utils/nfsdctl/nfsd_netlink.h b/utils/nfsdctl/nfsd_netlink.h
index 24c86dbc7ed5..887cbd12b695 100644
--- a/utils/nfsdctl/nfsd_netlink.h
+++ b/utils/nfsdctl/nfsd_netlink.h
@@ -70,6 +70,14 @@ enum {
NFSD_A_SERVER_SOCK_MAX = (__NFSD_A_SERVER_SOCK_MAX - 1)
};
+enum {
+ NFSD_A_POOL_MODE_MODE = 1,
+ NFSD_A_POOL_MODE_NPOOLS,
+
+ __NFSD_A_POOL_MODE_MAX,
+ NFSD_A_POOL_MODE_MAX = (__NFSD_A_POOL_MODE_MAX - 1)
+};
+
enum {
NFSD_CMD_RPC_STATUS_GET = 1,
NFSD_CMD_THREADS_SET,
@@ -78,6 +86,8 @@ enum {
NFSD_CMD_VERSION_GET,
NFSD_CMD_LISTENER_SET,
NFSD_CMD_LISTENER_GET,
+ NFSD_CMD_POOL_MODE_SET,
+ NFSD_CMD_POOL_MODE_GET,
__NFSD_CMD_MAX,
NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
diff --git a/utils/nfsdctl/nfsdctl.8 b/utils/nfsdctl/nfsdctl.8
index 7f5d981042a5..d55862cc1fc1 100644
--- a/utils/nfsdctl/nfsdctl.8
+++ b/utils/nfsdctl/nfsdctl.8
@@ -2,12 +2,12 @@
.\" Title: nfsdctl
.\" Author: Jeff Layton
.\" Generator: Asciidoctor 2.0.20
-.\" Date: 2024-04-23
+.\" Date: 2024-06-13
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
-.TH "NFSDCTL" "8" "2024-04-23" "\ \&" "\ \&"
+.TH "NFSDCTL" "8" "2024-06-13" "\ \&" "\ \&"
.ie \n(.g .ds Aq \(aq
.el .ds Aq '
.ss \n[.ss] 0
@@ -176,6 +176,25 @@ all minorversions for that major version.
.fi
.if n .RE
.RE
+.sp
+\fBpool\-mode\fP
+.RS 4
+Get/set the host\(cqs pool mode. This will cause the server to start threads
+that are pinned to either the CPU or the NUMA node. This can only be set
+when there are no nfsd threads running.
+.sp
+.if n .RS 4
+.nf
+.fam C
+The available options are:
+ global: single large pool
+ percpu: pool per CPU
+ pernode: pool per NUMA node
+ auto: choose a mode based on host configuration
+.fam
+.fi
+.if n .RE
+.RE
.SH "EXAMPLES"
.sp
Start the server with the settings in nfs.conf:
diff --git a/utils/nfsdctl/nfsdctl.adoc b/utils/nfsdctl/nfsdctl.adoc
index b2f00c11829e..03c5c281d92e 100644
--- a/utils/nfsdctl/nfsdctl.adoc
+++ b/utils/nfsdctl/nfsdctl.adoc
@@ -91,6 +91,18 @@ Each subcommand can also accept its own set of options and arguments. The
The minorversion field is optional. If not given, it will disable or enable
all minorversions for that major version.
+*pool-mode*::
+
+ Get/set the host's pool mode. This will cause the server to start threads
+ that are pinned to either the CPU or the NUMA node. This can only be set
+ when there are no nfsd threads running.
+
+ The available options are:
+ global: single large pool
+ percpu: pool per CPU
+ pernode: pool per NUMA node
+ auto: choose a mode based on host configuration
+
== EXAMPLES
Start the server with the settings in nfs.conf:
diff --git a/utils/nfsdctl/nfsdctl.c b/utils/nfsdctl/nfsdctl.c
index 986c937b620d..55bf5cd21783 100644
--- a/utils/nfsdctl/nfsdctl.c
+++ b/utils/nfsdctl/nfsdctl.c
@@ -320,20 +320,20 @@ static void parse_threads_get(struct genlmsghdr *gnlh)
struct nlattr *a;
switch (nla_type(attr)) {
- case NFSD_A_SERVER_GRACETIME:
- printf("gracetime: %u\n", nla_get_u32(attr));
- break;
- case NFSD_A_SERVER_LEASETIME:
- printf("leasetime: %u\n", nla_get_u32(attr));
- break;
- case NFSD_A_SERVER_SCOPE:
- printf("scope: %s\n", nla_data(attr));
- break;
- case NFSD_A_SERVER_THREADS:
- pool_threads[i++] = nla_get_u32(attr);
- break;
- default:
- break;
+ case NFSD_A_SERVER_GRACETIME:
+ printf("gracetime: %u\n", nla_get_u32(attr));
+ break;
+ case NFSD_A_SERVER_LEASETIME:
+ printf("leasetime: %u\n", nla_get_u32(attr));
+ break;
+ case NFSD_A_SERVER_SCOPE:
+ printf("scope: %s\n", nla_data(attr));
+ break;
+ case NFSD_A_SERVER_THREADS:
+ pool_threads[i++] = nla_get_u32(attr);
+ break;
+ default:
+ break;
}
}
@@ -343,6 +343,26 @@ static void parse_threads_get(struct genlmsghdr *gnlh)
putchar('\n');
}
+static void parse_pool_mode_get(struct genlmsghdr *gnlh)
+{
+ struct nlattr *attr;
+ int rem;
+
+ nla_for_each_attr(attr, genlmsg_attrdata(gnlh, 0),
+ genlmsg_attrlen(gnlh, 0), rem) {
+ switch (nla_type(attr)) {
+ case NFSD_A_POOL_MODE_MODE:
+ printf("pool-mode: %s\n", nla_data(attr));
+ break;
+ case NFSD_A_POOL_MODE_NPOOLS:
+ printf("npools: %u\n", nla_get_u32(attr));
+ break;
+ default:
+ break;
+ }
+ }
+}
+
static int recv_handler(struct nl_msg *msg, void *arg)
{
struct genlmsghdr *gnlh = nlmsg_data(nlmsg_hdr(msg));
@@ -361,6 +381,9 @@ static int recv_handler(struct nl_msg *msg, void *arg)
case NFSD_CMD_LISTENER_GET:
parse_listener_get(gnlh);
break;
+ case NFSD_CMD_POOL_MODE_GET:
+ parse_pool_mode_get(gnlh);
+ break;
default:
break;
}
@@ -1158,6 +1181,95 @@ static int listener_func(struct nl_sock *sock, int argc, char ** argv)
return 0;
}
+static int pool_mode_doit(struct nl_sock *sock, int cmd, const char *pool_mode)
+{
+ struct genlmsghdr *ghdr;
+ struct nlmsghdr *nlh;
+ struct nl_msg *msg;
+ struct nl_cb *cb;
+ int ret;
+
+ msg = netlink_msg_alloc(sock);
+ if (!msg)
+ return 1;
+
+ nlh = nlmsg_hdr(msg);
+ if (cmd == NFSD_CMD_POOL_MODE_SET) {
+ if (pool_mode)
+ nla_put_string(msg, NFSD_A_POOL_MODE_MODE, pool_mode);
+ }
+ ghdr = nlmsg_data(nlh);
+ ghdr->cmd = cmd;
+
+ cb = nl_cb_alloc(NL_CB_CUSTOM);
+ if (!cb) {
+ fprintf(stderr, "failed to allocate netlink callbacks\n");
+ ret = 1;
+ goto out;
+ }
+
+ ret = nl_send_auto(sock, msg);
+ if (ret < 0) {
+ fprintf(stderr, "send failed (%d)!\n", ret);
+ goto out_cb;
+ }
+
+ ret = 1;
+ nl_cb_err(cb, NL_CB_CUSTOM, error_handler, &ret);
+ nl_cb_set(cb, NL_CB_FINISH, NL_CB_CUSTOM, finish_handler, &ret);
+ nl_cb_set(cb, NL_CB_ACK, NL_CB_CUSTOM, ack_handler, &ret);
+ nl_cb_set(cb, NL_CB_VALID, NL_CB_CUSTOM, recv_handler, NULL);
+
+ while (ret > 0)
+ nl_recvmsgs(sock, cb);
+ if (ret < 0) {
+ fprintf(stderr, "Error: %s\n", strerror(-ret));
+ ret = 1;
+ }
+out_cb:
+ nl_cb_put(cb);
+out:
+ nlmsg_free(msg);
+ return ret;
+}
+
+static void pool_mode_usage(void)
+{
+ printf("Usage: %s pool-mode { global | auto | percpu | pernode } ...\n", taskname);
+}
+
+static int pool_mode_func(struct nl_sock *sock, int argc, char **argv)
+{
+ uint8_t cmd = NFSD_CMD_POOL_MODE_GET;
+ const char *pool_mode = NULL;
+ int opt;
+
+ optind = 1;
+ while ((opt = getopt_long(argc, argv, "h", help_only_options, NULL)) != -1) {
+ switch (opt) {
+ case 'h':
+ pool_mode_usage();
+ return 0;
+ }
+ }
+
+ if (optind < argc) {
+ char **targv = &argv[optind];
+ int i;
+
+ cmd = NFSD_CMD_POOL_MODE_SET;
+
+ /* empty string? */
+ if (targv[i][0] == '\0') {
+ fprintf(stderr, "Invalid threads value %s.\n", targv[i]);
+ return 1;
+ }
+
+ pool_mode = targv[0];
+ }
+ return pool_mode_doit(sock, cmd, pool_mode);
+}
+
#define MAX_LISTENER_LEN (64 * 2 + 16)
static int
@@ -1249,7 +1361,7 @@ static int autostart_func(struct nl_sock *sock, int argc, char ** argv)
int *threads, grace, lease, idx, ret, opt;
struct conf_list *thread_str;
struct conf_list_node *n;
- char *scope;
+ char *scope, *pool_mode;
optind = 1;
while ((opt = getopt_long(argc, argv, "h", help_only_options, NULL)) != -1) {
@@ -1262,6 +1374,13 @@ static int autostart_func(struct nl_sock *sock, int argc, char ** argv)
read_nfsd_conf();
+ pool_mode = conf_get_str("nfsd", "pool-mode");
+ if (pool_mode) {
+ ret = pool_mode_doit(sock, NFSD_CMD_POOL_MODE_SET, pool_mode);
+ if (ret)
+ return ret;
+ }
+
ret = fetch_nfsd_versions(sock);
if (ret)
return ret;
@@ -1312,6 +1431,7 @@ enum nfsdctl_commands {
NFSDCTL_VERSION,
NFSDCTL_LISTENER,
NFSDCTL_AUTOSTART,
+ NFSDCTL_POOL_MODE,
};
static int parse_command(char *str)
@@ -1326,6 +1446,8 @@ static int parse_command(char *str)
return NFSDCTL_LISTENER;
if (!strcmp(str, "autostart"))
return NFSDCTL_AUTOSTART;
+ if (!strcmp(str, "pool-mode"))
+ return NFSDCTL_POOL_MODE;
return -1;
}
@@ -1337,6 +1459,7 @@ static nfsdctl_func func[] = {
[NFSDCTL_VERSION] = version_func,
[NFSDCTL_LISTENER] = listener_func,
[NFSDCTL_AUTOSTART] = autostart_func,
+ [NFSDCTL_POOL_MODE] = pool_mode_func,
};
static void usage(void)
@@ -1348,6 +1471,7 @@ static void usage(void)
printf(" -d | --debug=NUM enable debugging\n");
printf(" -V | --version print version info\n");
printf(" commands:\n");
+ printf(" pool-mode get/set host pool mode setting\n");
printf(" listener get/set listener info\n");
printf(" version get/set supported NFS versions\n");
printf(" threads get/set nfsd thread settings\n");
--
2.45.2
