On Thu, Dec 12, 2024 at 04:33:51PM +0100, Emmanuel Florac wrote:
> Le Sat, 30 Nov 2024 09:14:20 +1100
> Dave Chinner <david@xxxxxxxxxxxxx> écrivait:
>
> > > xfs_quota -x -c "limit -p bhard=30000g 10" /mnt/raid
> >
> > That should set it up appropriately, hence the need to check if it
> > has actually been set up correctly on disk.
> >
>
> Unfortunately in the meantime the users did some cleanup, therefore the
> displayed information is coherent again (as there is more free space on
> the filesystem as a whole as any remaining allocated quota).
>
> xfs_quota -x -c "report -p"
> Project quota on /mnt/raid (/dev/mapper/vg0-raid)
> Blocks
> Project ID Used Soft Hard Warn/Grace
>
> ----------- ------------------------------------------------------
>
> <snip>
> #40 10795758244 0 16106127360 00 [--------]
>
>
> > > > Output of df and a project quota report showing usage and limits
> > > > would be useful here.
>
> looking at the corresponding folder :
>
> /dev/mapper/vg0-raid 15T 11T 5,0T 68% /mnt/raid/pad
>
>
> du -s /mnt/raid/pad
> 10795758244 /mnt/raid/pad
>
> # find /mnt/raid/pad -print | wc -l
> 39086
>
> > > > Then, for each of the top level project directories you are
> > > > querying with df, also run `xfs_io -rxc "stat" <dir>` and post
> > > > the output. This will tell us if the project quota is set up
> > > > correctly for df to report quota limits for them.
> > > >
>
> Starting with "pad" :
>
> # xfs_io -rxc "stat" pad
> fd.path = "."
> fd.flags = non-sync,non-direct,read-only
> stat.ino = 6442662464
> stat.type = directory
> stat.size = 4096
> stat.blocks = 16
> fsxattr.xflags = 0x200 \[--------P--------\]
> fsxattr.projid = 40
> fsxattr.extsize = 0
> fsxattr.cowextsize = 0
> fsxattr.nextents = 2
> fsxattr.naextents = 0
> dioattr.mem = 0x200
> dioattr.miniosz = 512
> dioattr.maxiosz = 2147483136
>
> # xfs_io -rxc "stat" rush
> fd.path = "."
> fd.flags = non-sync,non-direct,read-only
> stat.ino = 142
> stat.type = directory
> stat.size = 283
> stat.blocks = 0
> fsxattr.xflags = 0x200 \[--------P--------\]
> fsxattr.projid = 10
> fsxattr.extsize = 0
> fsxattr.cowextsize = 0
> fsxattr.nextents = 0
> fsxattr.naextents = 0
> dioattr.mem = 0x200
> dioattr.miniosz = 512
> dioattr.maxiosz = 2147483136
>
> # xfs_io -rxc "stat" labo
> fd.path = "."
> fd.flags = non-sync,non-direct,read-only
> stat.ino = 2147695168
> stat.type = directory
> stat.size = 310
> stat.blocks = 0
> fsxattr.xflags = 0x200 \[--------P--------\]
> fsxattr.projid = 20
> fsxattr.extsize = 0
> fsxattr.cowextsize = 0
> fsxattr.nextents = 0
> fsxattr.naextents = 0
> dioattr.mem = 0x200
> dioattr.miniosz = 512
> dioattr.maxiosz = 2147483136
>
> # xfs_io -rxc "stat" prods
> fd.path = "."
> fd.flags = non-sync,non-direct,read-only
> stat.ino = 4295178816
> stat.type = directory
> stat.size = 319
> stat.blocks = 0
> fsxattr.xflags = 0x200 \[--------P--------\]
> fsxattr.projid = 30
> fsxattr.extsize = 0
> fsxattr.cowextsize = 0
> fsxattr.nextents = 0
> fsxattr.naextents = 0
> dioattr.mem = 0x200
> dioattr.miniosz = 512
> dioattr.maxiosz = 2147483136
>
> > > > It would also be useful to know if the actual quota usage is
> > > > correct
> > > > - having the output of `du -s /mnt/raid/project1` to count the
> > > > blocks and `find /mnt/raid/project1 -print |wc -l` to count the
> > > > files in quota controlled directories. That'll give us some idea
> > > > if there's a quota accounting issue.
> >
> > iAnother thought occurred to me - can you also check that
> > /etc/projid and /etc/projects is similar on all machines, and post
> > the contents of them from the bad machine?
> >
>
> Hum, actually they didn't set up neither projid nor projects. Of course
> I did create these during my tests, but could this be the culprit ?
Does this recreate the symptoms?
# mkfs.xfs -f /dev/sda
meta-data=/dev/sda isize=512 agcount=4, agsize=1298176 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=1, sparse=1, rmapbt=1
= reflink=1 bigtime=1 inobtcount=1 nrext64=1
= exchange=0 metadir=0
data = bsize=4096 blocks=5192704, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0, ftype=1, parent=0
log =internal log bsize=4096 blocks=16384, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
= rgcount=0 rgsize=0 extents
Discarding blocks...Done.
# mount /dev/sda /mnt -o prjquota
# xfs_quota -x -c 'limit -p bhard=2G 55' /mnt
# mkdir /mnt/dir
# xfs_io -c 'chproj 55' -c 'chattr +P' -c 'stat -vvvv' /mnt/dir
fd.path = "/mnt/dir"
fd.flags = non-sync,non-direct,read-write
stat.ino = 132
stat.type = directory
stat.size = 6
stat.blocks = 0
stat.atime = Thu Dec 12 12:07:53 2024
stat.mtime = Thu Dec 12 12:07:53 2024
stat.ctime = Thu Dec 12 12:08:12 2024
fsxattr.xflags = 0x200 [proj-inherit]
fsxattr.projid = 55
fsxattr.extsize = 0
fsxattr.cowextsize = 0
fsxattr.nextents = 0
fsxattr.naextents = 0
dioattr.mem = 0x200
dioattr.miniosz = 512
dioattr.maxiosz = 2147483136
# df /mnt /mnt/dir
Filesystem Size Used Avail Use% Mounted on
/dev/sda 20G 420M 20G 3% /mnt
/dev/sda 2.0G 0 2.0G 0% /mnt
# fallocate -l 19g /mnt/a
# df /mnt /mnt/dir
Filesystem Size Used Avail Use% Mounted on
/dev/sda 20G 20G 345M 99% /mnt
/dev/sda 2.0G 0 2.0G 0% /mnt
Clearly, df should be reporting 345M available for both cases, since we
haven't actually used any of project 55's blocks.
# xfs_io -f -c 'pwrite -S 0x59 0 1m' -c fsync -c 'stat -vvvv' /mnt/dir/fork
wrote 1048576/1048576 bytes at offset 0
1 MiB, 256 ops; 0.0008 sec (1.121 GiB/sec and 293915.0402 ops/sec)
fd.path = "/mnt/dir/fork"
fd.flags = non-sync,non-direct,read-write
stat.ino = 134
stat.type = regular file
stat.size = 1048576
stat.blocks = 2048
stat.atime = Thu Dec 12 12:11:06 2024
stat.mtime = Thu Dec 12 12:11:06 2024
stat.ctime = Thu Dec 12 12:11:06 2024
fsxattr.xflags = 0x0 []
fsxattr.projid = 55
fsxattr.extsize = 0
fsxattr.cowextsize = 0
fsxattr.nextents = 1
fsxattr.naextents = 0
dioattr.mem = 0x200
dioattr.miniosz = 512
dioattr.maxiosz = 2147483136
# df /mnt /mnt/dir
Filesystem Size Used Avail Use% Mounted on
/dev/sda 20G 20G 344M 99% /mnt
/dev/sda 2.0G 1.0M 2.0G 1% /mnt
I think this behavior comes from xfs_fill_statvfs_from_dquot, which does
this:
limit = blkres->softlimit ?
blkres->softlimit :
blkres->hardlimit;
if (limit && statp->f_blocks > limit) {
statp->f_blocks = limit;
statp->f_bfree = statp->f_bavail =
(statp->f_blocks > blkres->reserved) ?
(statp->f_blocks - blkres->reserved) : 0;
}
I think the f_bfree/f_bavail assignment is wrong because it doesn't
handle the case where f_bfree was less than (limit - reserved).
if (limit) {
uint64_t remaining = 0;
if (statp->f_blocks > limit)
statp->f_blocks = limit;
if (limit > blkres->reserved)
remaining = limit - blkres->reserved;
statp->f_bfree = min(statp->f_bfree, remaining);
statp->f_bavail = min(statp->f_bavail, remaining);
}
This fixes the df output a bit:
# df /mnt /mnt/dir
Filesystem Size Used Avail Use% Mounted on
/dev/sda 20G 20G 344M 99% /mnt
/dev/sda 2.0G 1.7G 344M 84% /mnt
Though the "used" column is nonsense now. But I guess that's why statfs
only defines total blocks and free/available blocks.
--D
> --
> ------------------------------------------------------------------------
> Emmanuel Florac | Direction technique
> ------------------------------------------------------------------------
> https://intellique.com
> +33 6 16 30 15 95
> ------------------------------------------------------------------------
>
