We have incremental mode for supporting hot plug of devices. However,
we don't support hot-unplug. This is something we need in order to
enable automatic device replacement. When implementing the support
though, I found that we had a problem in that the device special file is
already gone (well, not gone, but both open and lstat fail on the device
special file, which really makes Manage_subdevs unhappy) by the time
udev calls us to remove the device (even if I created a new udev rules
file with a very low number this was still true). So, along with adding
the code shamelessly stolen from Hawrylewicz with modifications to make
it work on non-container based arrays, I had to modify Manage_subdevs to
work on sysfs subdevice entries in the fail and remove cases. Anyway,
with this in place, and with the modified udev rules file in place,
hot-unplug now works (or at least it does in my testing, I didn't try a
container device, I'll leave tweaking that to Dan or someone else from
Intel, although it should more or less work, I don't know the Intel
metadata rules and so didn't try to make it work myself). And by
changing Manage_subdevs to use either a valid device special file or
sysfs entries, it will work on the command line with virtually any
kernel, and work from a udev rules file on kernels recent enough to have
the state entry in sysfs subdevs directories.
Oh, and Neil should review my man page additions to see if they are
sufficient for his tastes. I didn't get real wordy about the support,
it's pretty straight forward.
--
Doug Ledford <dledford@xxxxxxxxxx>
GPG KeyID: CFBFF194
http://people.redhat.com/dledford
Infiniband specific RPMs available at
http://people.redhat.com/dledford/Infiniband
commit 3212dcca06f81017ec9a50c7b5d4eeb34bedaab6
Author: Doug Ledford <dledford@xxxxxxxxxx>
Date: Mon Apr 5 12:32:08 2010 -0400
Initial implementation of incremental remove support
Signed-off-by: Doug Ledford <dledford@xxxxxxxxxx>
diff --git a/Incremental.c b/Incremental.c
index 7ad648a..d32a8e5 100644
--- a/Incremental.c
+++ b/Incremental.c
@@ -843,3 +843,39 @@ int Incremental_container(struct supertype *st, char *devname, int verbose,
map_unlock(&map);
return 0;
}
+
+/*
+ * IncrementalRemove - Attempt to see if the passed in device belongs to any
+ * raid arrays, and if so first fail (if needed) and then remove the device.
+ *
+ * @devname - The device we want to remove
+ *
+ * Special note: We would like to just use Managedevs to fail/remove the
+ * device, but unfortunately, by the time we are called via udev, the device
+ * special file is already gone, and so we can't stat the device and se we
+ * don't have the right rdev value to use in the ioctls. So, we use the
+ * sysfs method of device removal instead, but since that's not gauranteed
+ * to work depending on the version of kernel we run on, try to use the
+ * ioctl method first and only fallback if we don't have a valid device
+ * special file. That way we can support operation manually on older kernels
+ * even if we won't be able to do this automatically via udev on older
+ * kernels.
+ */
+int IncrementalRemove(char *devname, int verbose)
+{
+ char mddev[100] = "/dev/";
+ int mdfd;
+ struct mddev_dev_s devlist;
+
+ strncpy(mddev + 5, devname, sizeof(mddev) - 5);
+ if (mdstat_check_active(mddev + 5))
+ return 1;
+ if ((mdfd = open_mddev(mddev, 0)) < 0)
+ return 1;
+ memset(&devlist, 0, sizeof(devlist));
+ devlist.devname = devname;
+ devlist.disposition = 'f';
+ Manage_subdevs(mddev, mdfd, &devlist, verbose);
+ devlist.disposition = 'r';
+ return Manage_subdevs(mddev, mdfd, &devlist, verbose);
+}
diff --git a/Manage.c b/Manage.c
index f848d8b..6539eda 100644
--- a/Manage.c
+++ b/Manage.c
@@ -346,6 +346,9 @@ int Manage_subdevs(char *devname, int fd,
mdu_disk_info_t disc;
unsigned long long array_size;
mddev_dev_t dv, next = NULL;
+ struct mdinfo *mdi = NULL;
+ struct mdinfo *dev = NULL;
+ char sys_name[20] = "dev-";
struct stat stb;
int j, jnext = 0;
int tfd;
@@ -443,16 +446,43 @@ int Manage_subdevs(char *devname, int fd,
if (jnext == 0)
continue;
} else {
+ /*
+ * For fail/remove operations, allow the disk
+ * to be completely missing, use name matching
+ * to a device in our sysfs entries to
+ * suffice. For add we need a valid block device.
+ * Leave this loop one of three ways:
+ * 1) tfd < 0 and dev is set to our device
+ * 2) tfd >= 0 and dev is NULL
+ * 3) failed to find suitable device and return
+ */
j = 0;
tfd = dev_open(dv->devname, O_RDONLY);
- if (tfd < 0 && dv->disposition == 'r' &&
- lstat(dv->devname, &stb) == 0)
- /* Be happy, the lstat worked, that is
- * enough for --remove
- */
- ;
- else {
+ if (tfd < 0 && dv->disposition != 'a') {
+ strcpy(&sys_name[4],
+ strrchr(dv->devname, '/') + 1);
+ mdi = sysfs_read(fd, 0,
+ GET_DEVS | KEEP_GONE_DEVS);
+ if (!mdi) {
+ fprintf(stderr, Name ": can't open %s "
+ "and can't read sysfs info\n",
+ dv->devname);
+ return 1;
+ }
+ for (dev = mdi->devs; dev; dev = dev->next) {
+ if (strcmp(sys_name, dev->sys_name))
+ continue;
+ break;
+ }
+ if (!dev) {
+ fprintf(stderr, Name ": can't open %s "
+ "and %s not listed in sysfs\n",
+ dv->devname, sys_name);
+ sysfs_free(mdi);
+ return 1;
+ }
+ } else {
if (tfd < 0 || fstat(tfd, &stb) != 0) {
fprintf(stderr, Name ": cannot find %s: %s\n",
dv->devname, strerror(errno));
@@ -461,12 +491,12 @@ int Manage_subdevs(char *devname, int fd,
return 1;
}
close(tfd);
- }
- if ((stb.st_mode & S_IFMT) != S_IFBLK) {
- fprintf(stderr, Name ": %s is not a "
- "block device.\n",
- dv->devname);
- return 1;
+ if ((stb.st_mode & S_IFMT) != S_IFBLK) {
+ fprintf(stderr, Name ": %s is not a "
+ "block device.\n",
+ dv->devname);
+ return 1;
+ }
}
}
switch(dv->disposition){
@@ -790,26 +820,36 @@ int Manage_subdevs(char *devname, int fd,
return 1;
}
}
- /* FIXME check that it is a current member */
- err = ioctl(fd, HOT_REMOVE_DISK, (unsigned long)stb.st_rdev);
- if (err && errno == ENODEV) {
+ /* stb.st_rdev is only valid if we have a tfd that
+ * does not indicate an error on attempt to open
+ * the devname
+ */
+ if (tfd >= 0)
+ err = ioctl(fd, HOT_REMOVE_DISK,
+ (unsigned long)stb.st_rdev);
+ if (tfd < 0 || (err && errno == ENODEV)) {
/* Old kernels rejected this if no personality
* registered */
- struct mdinfo *sra = sysfs_read(fd, 0, GET_DEVS);
- struct mdinfo *dv = NULL;
- if (sra)
- dv = sra->devs;
- for ( ; dv ; dv=dv->next)
- if (dv->disk.major == major(stb.st_rdev) &&
- dv->disk.minor == minor(stb.st_rdev))
+ if (!mdi) {
+ strcpy(&sys_name[4],
+ strrchr(dv->devname, '/') + 1);
+ mdi = sysfs_read(fd, 0, GET_DEVS |
+ KEEP_GONE_DEVS);
+ if (mdi)
+ dev = mdi->devs;
+ for ( ; dev ; dev=dev->next) {
+ if (strcmp(sys_name, dev->sys_name))
+ continue;
break;
- if (dv)
- err = sysfs_set_str(sra, dv,
+ }
+ }
+ if (dev)
+ err = sysfs_set_str(mdi, dev,
"state", "remove");
else
err = -1;
- if (sra)
- sysfs_free(sra);
+ if (mdi)
+ sysfs_free(mdi);
}
if (err) {
fprintf(stderr, Name ": hot remove failed "
@@ -844,11 +884,18 @@ int Manage_subdevs(char *devname, int fd,
case 'f': /* set faulty */
/* FIXME check current member */
- if (ioctl(fd, SET_DISK_FAULTY, (unsigned long) stb.st_rdev)) {
+ if ((tfd >= 0 && ioctl(fd, SET_DISK_FAULTY,
+ (unsigned long) stb.st_rdev)) ||
+ (tfd < 0 && sysfs_set_str(mdi, dev, "state",
+ "faulty"))) {
fprintf(stderr, Name ": set device faulty failed for %s: %s\n",
dnprintable, strerror(errno));
+ if (mdi)
+ sysfs_free(mdi);
return 1;
- }
+ }
+ if (mdi)
+ sysfs_free(mdi);
if (verbose >= 0)
fprintf(stderr, Name ": set %s faulty in %s\n",
dnprintable, devname);
diff --git a/ReadMe.c b/ReadMe.c
index ba2215c..acaeace 100644
--- a/ReadMe.c
+++ b/ReadMe.c
@@ -86,11 +86,12 @@ char Version[] = Name " - v3.1.2-dledford - 30th March 2010\n";
* At the time if writing, there is only minimal support.
*/
-char short_options[]="-ABCDEFGIQhVXWZ:vqbc:i:l:p:m:n:x:u:c:d:z:U:N:sarfRSow1tye:";
+char short_options[]=
+ "-ABCDEFGIQhVXWZ:vqbc:i:l:p:m:n:x:u:c:d:z:U:N:sarfRSow1tye:";
char short_bitmap_options[]=
- "-ABCDEFGIQhVXWZ:vqb:c:i:l:p:m:n:x:u:c:d:z:U:N:sarfRSow1tye:";
+ "-ABCDEFGIQhVXWZ:vqb:c:i:l:p:m:n:x:u:c:d:z:U:N:sarfRSow1tye:";
char short_bitmap_auto_options[]=
- "-ABCDEFGIQhVXWZ:vqb:c:i:l:p:m:n:x:u:c:d:z:U:N:sa:rfRSow1tye:";
+ "-ABCDEFGIQhVXWZ:vqb:c:i:l:p:m:n:x:u:c:d:z:U:N:sa:rfRSow1tye:";
struct option long_options[] = {
{"manage", 0, 0, '@'},
@@ -213,7 +214,7 @@ char Help[] =
" mdadm --grow options device\n"
" resize/reshape an active array\n"
" mdadm --incremental device\n"
-" add a device to an array as appropriate\n"
+" add/remove a device to/from an array as appropriate\n"
" mdadm --monitor options...\n"
" Monitor one or more array for significant changes.\n"
" mdadm device options...\n"
@@ -535,20 +536,26 @@ char Help_grow[] =
;
char Help_incr[] =
-"Usage: mdadm --incremental [-Rqrs] device\n"
+"Usage: mdadm --incremental [-Rqrsf] device\n"
"\n"
"This usage allows for incremental assembly of md arrays. Devices can be\n"
"added one at a time as they are discovered. Once an array has all expected\n"
"devices, it will be started.\n"
"\n"
-"Options that are valid with incremental assembly (-I --incremental) more are:\n"
-" --run -R : run arrays as soon as a minimal number of devices are\n"
+"Optionally, the process can be reversed by using the fail option.\n"
+"When fail mode is invoked, mdadm will see if the device belongs to an array\n"
+"and then both fail (if needed) and remove the device from that array.\n"
+"\n"
+"Options that are valid with incremental assembly (-I --incremental) are:\n"
+" --run -R : Run arrays as soon as a minimal number of devices are\n"
" : present rather than waiting for all expected.\n"
" --quiet -q : Don't print any information messages, just errors.\n"
" --rebuild -r : Rebuild the 'map' file that mdadm uses for tracking\n"
" : partial arrays.\n"
" --scan -s : Use with -R to start any arrays that have the minimal\n"
" : required number of devices, but are not yet started.\n"
+" --fail -f : First fail (if needed) and then remove device from\n"
+" : any array that it is a member of.\n"
;
char Help_config[] =
diff --git a/mdadm.8 b/mdadm.8
index 4edfc41..eaf9155 100644
--- a/mdadm.8
+++ b/mdadm.8
@@ -135,7 +135,11 @@ This provides a convenient interface to a
.I hot-plug
system. As each device is detected,
.I mdadm
-has a chance to include it in some array as appropriate.
+has a chance to include it in some array as appropriate. Optionally,
+with the
+.I \-\-fail
+flag is passed in then we will remove the device from any active array
+instead of adding it.
If a
.B CONTAINER
@@ -189,7 +193,7 @@ Change the size or shape of an active array.
.TP
.BR \-I ", " \-\-incremental
-Add a single device into an appropriate array, and possibly start the array.
+Add/remove a single device to/from an appropriate array, and possibly start the array.
.TP
.B \-\-auto-detect
@@ -1235,6 +1239,12 @@ in
.B mdadm.conf
as requiring an external bitmap, that bitmap will be attached first.
+.TP
+.BR \-\-fail ", " \-f
+This allows the hot-plug system to remove devices that have fully disappeared
+from the kernel. It will first fail and then remove the device from any
+array it belongs to.
+
.SH For Monitor mode:
.TP
.BR \-m ", " \-\-mail
@@ -2141,6 +2151,10 @@ Usage:
.I component-device
.HP 12
Usage:
+.B mdadm \-\-incremental \-\-fail
+.I component-device
+.HP 12
+Usage:
.B mdadm \-\-incremental \-\-rebuild
.HP 12
Usage:
@@ -2153,6 +2167,11 @@ passed to
.B "mdadm \-\-incremental"
to be conditionally added to an appropriate array.
+Conversely, it can also be used with the
+.B \-\-fail
+flag to do just the opposite and find whatever array a particular device
+is part of and remove the device from the array.
+
If the device passed is a
.B CONTAINER
device created by a previous call to
diff --git a/mdadm.c b/mdadm.c
index d5e34c0..cd6fd8f 100644
--- a/mdadm.c
+++ b/mdadm.c
@@ -124,6 +124,7 @@ int main(int argc, char *argv[])
ident.name[0] = 0;
ident.container = NULL;
ident.member = NULL;
+ ident.member_index = -1;
while ((option_index = -1) ,
(opt=getopt_long(argc, argv,
@@ -774,6 +775,9 @@ int main(int argc, char *argv[])
devmode = 'r';
continue;
case O(MANAGE,'f'): /* set faulty */
+ case O(INCREMENTAL,'f'): /* r for incremental is taken, use f
+ * even though we will both fail and
+ * remove the device */
devmode = 'f';
continue;
case O(INCREMENTAL,'R'):
@@ -1517,6 +1521,11 @@ int main(int argc, char *argv[])
": --incremental --scan meaningless without --run.\n");
break;
}
+ if (devmode == 'f') {
+ fprintf(stderr, Name
+ ": --incremental --scan --fail not supported.\n");
+ break;
+ }
rv = IncrementalScan(verbose);
}
if (!devlist) {
@@ -1533,6 +1542,10 @@ int main(int argc, char *argv[])
rv = 1;
break;
}
+ if (devmode == 'f') {
+ rv = IncrementalRemove(devlist->devname, verbose-quiet);
+ break;
+ }
rv = Incremental(devlist->devname, verbose-quiet, runstop,
ss, homehost, require_homehost, autof);
break;
diff --git a/mdadm.h b/mdadm.h
index d8ab85f..c113d0f 100644
--- a/mdadm.h
+++ b/mdadm.h
@@ -315,6 +315,7 @@ typedef struct mddev_ident_s {
* of some other entry.
*/
char *member; /* subarray within a container */
+ int member_index; /* subarray index within a container */
struct mddev_ident_s *next;
union {
@@ -355,6 +356,10 @@ struct mdstat_ent {
int raid_disks;
int chunk_size;
char * metadata_version;
+ struct dev_member {
+ char *name;
+ struct dev_member *next;
+ } *members;
struct mdstat_ent *next;
};
@@ -363,6 +368,7 @@ extern void free_mdstat(struct mdstat_ent *ms);
extern void mdstat_wait(int seconds);
extern void mdstat_wait_fd(int fd, const sigset_t *sigmask);
extern int mddev_busy(int devnum);
+extern int mdstat_check_active(char *devname);
struct map_ent {
struct map_ent *next;
@@ -404,6 +410,7 @@ enum sysfs_read_flags {
GET_STATE = (1 << 13),
GET_ERROR = (1 << 14),
SKIP_GONE_DEVS = (1 << 15),
+ KEEP_GONE_DEVS = (1 << 16),
};
/* If fd >= 0, get the array it is open on,
@@ -817,6 +824,7 @@ extern int Incremental_container(struct supertype *st, char *devname,
int trustworthy);
extern void RebuildMap(void);
extern int IncrementalScan(int verbose);
+extern int IncrementalRemove(char *devname, int verbose);
extern int CreateBitmap(char *filename, int force, char uuid[16],
unsigned long chunksize, unsigned long daemon_sleep,
diff --git a/mdstat.c b/mdstat.c
index 4a9f370..81d2212 100644
--- a/mdstat.c
+++ b/mdstat.c
@@ -83,6 +83,45 @@
#include <sys/select.h>
#include <ctype.h>
+static void free_member_devnames(struct dev_member **m)
+{
+ struct dev_member *t;
+ if (!*m)
+ return;
+ while(*m) {
+ t = *m;
+ *m = (*m)->next;
+ if (t->name)
+ free(t->name);
+ free(t);
+ }
+ *m = NULL;
+}
+
+static struct dev_member *add_member_devname(struct dev_member **m, char *name)
+{
+ struct dev_member *new;
+ char *t;
+
+ if (!m || !name)
+ return NULL;
+
+ new = malloc(sizeof(*new));
+ if (!new)
+ return NULL;
+ if ((t = strchr(name, '[')) == NULL)
+ {
+ /* not a device */
+ free(new);
+ return *m;
+ }
+ new->name = strndup(name, t - name);
+ new->next = *m;
+ *m = new;
+
+ return new;
+}
+
void free_mdstat(struct mdstat_ent *ms)
{
while (ms) {
@@ -91,6 +130,7 @@ void free_mdstat(struct mdstat_ent *ms)
if (ms->level) free(ms->level);
if (ms->pattern) free(ms->pattern);
if (ms->metadata_version) free(ms->metadata_version);
+ if (ms->members) free_member_devnames(&ms->members);
t = ms;
ms = ms->next;
free(t);
@@ -159,6 +199,7 @@ struct mdstat_ent *mdstat_read(int hold, int start)
ent->raid_disks = 0;
ent->chunk_size = 0;
ent->devcnt = 0;
+ ent->members = NULL;
ent->dev = strdup(line);
ent->devnum = devnum;
@@ -170,15 +211,23 @@ struct mdstat_ent *mdstat_read(int hold, int start)
ent->active = 1;
else if (strcmp(w, "inactive")==0)
ent->active = 0;
- else if (ent->active >=0 &&
+ else if (ent->active > 0 &&
ent->level == NULL &&
w[0] != '(' /*readonly*/) {
ent->level = strdup(w);
in_devs = 1;
} else if (in_devs && strcmp(w, "blocks")==0)
in_devs = 0;
- else if (in_devs) {
+ else if (in_devs || (ent->active == 0 && w[0] != '(' &&
+ w[l - 1] == ')')) {
+ if (isdigit(w[0]))
+ continue;
+ in_devs = 1;
ent->devcnt++;
+ if (!add_member_devname(&ent->members, w)) {
+ free_mdstat(ent);
+ break;
+ }
if (strncmp(w, "md", 2)==0) {
/* This has an md device as a component.
* If that device is already in the
@@ -310,3 +359,40 @@ int mddev_busy(int devnum)
free_mdstat(mdstat);
return me != NULL;
}
+
+/*
+ * Finds name of the active array holding this device
+ * @param[in] devname name of member device
+ * @param[out] devname name of array
+ *
+ * @return found (0), or
+ * not found, failure (1)
+ */
+
+int mdstat_check_active(char *devname)
+{
+ struct mdstat_ent *mdstat = mdstat_read(0, 0);
+ struct mdstat_ent *ent;
+ char *name;
+
+ if (!devname)
+ return 1;
+ name = strrchr(devname, '/');
+ if (name++ == NULL)
+ return 1;
+
+ for (ent = mdstat; ent; ent = ent->next) {
+ struct dev_member *m;
+ if (ent->active && (strstr(ent->metadata_version,"imsm") ||
+ strstr(ent->metadata_version,"ddf")))
+ /* only return container matches, not subarrays */
+ continue;
+ for (m = ent->members; m; m = m->next) {
+ if (!strcmp(m->name, name)) {
+ strcpy(devname, ent->dev);
+ return 0;
+ }
+ }
+ }
+ return 1;
+}
diff --git a/sysfs.c b/sysfs.c
index ebf9d8a..65dd848 100644
--- a/sysfs.c
+++ b/sysfs.c
@@ -273,13 +273,16 @@ struct mdinfo *sysfs_read(int fd, int devnum, unsigned long options)
strcpy(dbase, "block/dev");
if (load_sys(fname, buf)) {
- free(dev);
- if (options & SKIP_GONE_DEVS)
+ if (options & SKIP_GONE_DEVS) {
+ free(dev);
continue;
- else
+ } else if (options & KEEP_GONE_DEVS) {
+ dev->disk.major = dev->disk.minor = -1;
+ } else
goto abort;
- }
- sscanf(buf, "%d:%d", &dev->disk.major, &dev->disk.minor);
+ } else
+ sscanf(buf, "%d:%d", &dev->disk.major,
+ &dev->disk.minor);
/* special case check for block devices that can go 'offline' */
if (options & SKIP_GONE_DEVS) {
diff --git a/udev-md-raid.rules b/udev-md-raid.rules
index c9a4f0e..aff14fa 100644
--- a/udev-md-raid.rules
+++ b/udev-md-raid.rules
@@ -1,17 +1,16 @@
# do not edit this file, it will be overwritten on update
SUBSYSTEM!="block", GOTO="md_end"
-ACTION!="add|change", GOTO="md_end"
+ACTION!="add|change|remove", GOTO="md_end"
+ACTION=="remove", GOTO="md_remove"
ACTION=="change", GOTO="md_no_incr"
-# import data from a raid member and activate it
-#ENV{ID_FS_TYPE}=="linux_raid_member", IMPORT{program}="/sbin/mdadm --examine --export $tempnode", RUN+="/sbin/mdadm --incremental $env{DEVNAME}"
-# import data from a raid set
+# we are adding a raid member, activate it
+ENV{ID_FS_TYPE}=="linux_raid_member", RUN+="/sbin/mdadm -I $env{DEVNAME}"
LABEL="md_no_incr"
KERNEL!="md*", GOTO="md_end"
-# partitions have no md/{array_state,metadata_version}, but should not
-# for that reason be ignored.
+# partitions have no md/{array_state,metadata_version}
ENV{DEVTYPE}=="partition", GOTO="md_ignore_state"
# container devices have a metadata version of e.g. 'external:ddf' and
@@ -32,7 +31,12 @@ ENV{DEVTYPE}=="partition", ENV{MD_DEVNAME}=="*[0-9]", SYMLINK+="md/$env{MD_DEVNA
IMPORT{program}="/sbin/blkid -o udev -p $tempnode"
OPTIONS+="link_priority=100"
+OPTIONS+="watch"
ENV{ID_FS_USAGE}=="filesystem|other|crypto", ENV{ID_FS_UUID_ENC}=="?*", SYMLINK+="disk/by-uuid/$env{ID_FS_UUID_ENC}"
ENV{ID_FS_USAGE}=="filesystem|other", ENV{ID_FS_LABEL_ENC}=="?*", SYMLINK+="disk/by-label/$env{ID_FS_LABEL_ENC}"
+GOTO="md_end"
+
+LABEL="md_remove"
+ENV{ID_FS_TYPE}=="linux_raid_member", RUN+="/sbin/mdadm -If $env{DEVNAME}"
LABEL="md_end"
Attachment:
signature.asc
Description: OpenPGP digital signature
