On 2012-05-31 03:25, Boaz Harrosh wrote:
> If any one is interested the below is the behavior
> I'm striving for.
This patch looks reasonable to me.
Would you like me to commit it?
Benny
>
> ----
> From: Boaz Harrosh <bharrosh@xxxxxxxxxxx>
> Subject: [PATCH] pnfsd-exofs: Two clients must not write to the same RAID stripe
>
> If we have file redundancy RAID1/4/5/6 then two clients cannot
> write to the same stripe/region.
>
> We take care of this by giving out smaller regions of the file.
> Before any layout_get we make sure to recall the same exact
> region from any client. If a recall was issued we return
> NFS4ERR_RECALLCONFLICT. The client will come again later for
> it's layout.
>
> Meanwhile the fist client can flush data and release the
> layout. The next time the segment might be free and the
> lo_get succeed.
>
> It is very possible that multiple writers will fight and
> some clients will starve forever. But the smaller the
> region, and if the clients randomize a wait, it should
> statistically be OK.
> (We could manage a fairness queue. What about an lo_available
> notification)
>
> On the other hand a very small segment will hurt performance,
> so default size is now set to 8 stripes.
> TODO: Let segment size be set in sysfs.
>
> TODO:
> For debugging we always give out small segments. But we should
> only start giving out small segments on a shared file. The
> first/single writer should get a large seg as before.
>
> Signed-off-by: Boaz Harrosh <bharrosh@xxxxxxxxxxx>
> ---
> fs/exofs/export.c | 48 ++++++++++++++++++++++++++++++++++++++++++------
> 1 file changed, 42 insertions(+), 6 deletions(-)
>
> diff --git a/fs/exofs/export.c b/fs/exofs/export.c
> index c5712f3..a1f112f 100644
> --- a/fs/exofs/export.c
> +++ b/fs/exofs/export.c
> @@ -29,6 +29,9 @@
>
> #include "linux/nfsd/pnfs_osd_xdr_srv.h"
>
> +/* TODO: put in sysfs per sb */
> +const static unsigned sb_shared_num_stripes = 8;
> +
> static int exofs_layout_type(struct super_block *sb)
> {
> return LAYOUT_OSD2_OBJECTS;
> @@ -94,14 +97,27 @@ void ore_layout_2_pnfs_layout(struct pnfs_osd_layout *pl,
> }
> }
>
> -static void _align_io(struct ore_layout *layout, u64 *offset, u64 *length)
> +static bool _align_io(struct ore_layout *layout, struct nfsd4_layout_seg *lseg,
> + bool shared)
> {
> u64 stripe_size = (layout->group_width - layout->parity) *
> layout->stripe_unit;
> u64 group_size = stripe_size * layout->group_depth;
>
> - *offset = div64_u64(*offset, group_size) * group_size;
> - *length = group_size;
> + /* TODO: Don't ignore shared flag. Single writer can get a full group */
> + if (lseg->iomode != IOMODE_READ &&
> + (layout->parity || (layout->mirrors_p1 > 1))) {
> + /* RAID writes */
> + lseg->offset = div64_u64(lseg->offset, stripe_size) *
> + stripe_size;
> + lseg->length = stripe_size * sb_shared_num_stripes;
> + return true;
> + } else {
> + /* reads or no data redundancy */
> + lseg->offset = div64_u64(lseg->offset, group_size) * group_size;
> + lseg->length = group_size;
> + return false;
> + }
> }
>
> static enum nfsstat4 exofs_layout_get(
> @@ -116,15 +132,15 @@ static enum nfsstat4 exofs_layout_get(
> struct pnfs_osd_layout layout;
> __be32 *start;
> unsigned i;
> - bool in_recall;
> + bool in_recall, need_recall;
> enum nfsstat4 nfserr;
>
> EXOFS_DBGMSG("(0x%lx) REQUESTED offset=0x%llx len=0x%llx iomod=0x%x\n",
> inode->i_ino, res->lg_seg.offset,
> res->lg_seg.length, res->lg_seg.iomode);
>
> - _align_io(&sbi->layout, &res->lg_seg.offset, &res->lg_seg.length);
> - res->lg_seg.iomode = IOMODE_RW;
> + need_recall = _align_io(&sbi->layout, &res->lg_seg,
> + test_bit(OBJ_LAYOUT_IS_GIVEN, &oi->i_flags));
> res->lg_return_on_close = true;
> res->lg_lo_cookie = inode; /* Just for debug prints */
>
> @@ -132,6 +148,26 @@ static enum nfsstat4 exofs_layout_get(
> inode->i_ino, res->lg_seg.offset,
> res->lg_seg.length, res->lg_seg.iomode);
>
> + if (need_recall) {
> + int rc = cb_layout_recall(inode, IOMODE_RW, res->lg_seg.offset,
> + res->lg_seg.length, (void *)0x17);
> + switch (rc) {
> + case 0:
> + case -EAGAIN:
> + EXOFS_DBGMSG("(0x%lx) @@@ Sharing of RAID5/1 stripe\n",
> + inode->i_ino);
> + return NFS4ERR_RECALLCONFLICT;
> + default:
> + /* This is fine for now */
> + /* TODO: Fence object off */
> + EXOFS_DBGMSG("(0x%lx) !!!cb_layout_recall => %d\n",
> + inode->i_ino, rc);
> + /*fallthrough*/
> + case -ENOENT:
> + break;
> + }
> + }
> +
> /* skip opaque size, will be filled-in later */
> start = exp_xdr_reserve_qwords(xdr, 1);
> if (!start) {
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
