[cc fstests and the original author of data-only layer tests]
On Mon, Jul 29, 2024 at 12:22 AM Mike Baynton <mike@xxxxxxxxxxxx> wrote:
>
> The test only covers for files added, but not files undergoing any
> modification, including during their initial write. This generally means
> a technique such as renaming the file into the data-only area of the
> underlying filesystem is required.
>
> The defined behaviors are fairly minimal:
> * A file added to a data-only layer while mounted will not appear in
> the overlayfs via readdir or lookup, but it is safe for applications
> to attempt to do so.
> * A subsequently mounted overlayfs that includes redirects to the added
> files will be able to iterate and open the added files.
>
AFAIK, this is how all data-only overlayfs works, because the
data-only layer is always going to be a layer that is shared among
many overlayfs, so at any given time, there would be an online overlayfs
when blobs are added to the data-only layer to compose new images.
It is good to make this behavior known and explicit - I am just saying
that it is implied by the data-only layers features, because it would
have been useless otherwise.
> Signed-off-by: Mike Baynton <mike@xxxxxxxxxxxx>
> ---
> Looks like somewhere wrapping got added despite my best efforts with the
> patch on my last email. Sending patch on its own as well in case someone
> wants to actually apply/run it.
>
> tests/overlay/087 | 170 ++++++++++++++++++++++++++++++++++++++++++
> tests/overlay/087.out | 13 ++++
> 2 files changed, 183 insertions(+)
> create mode 100755 tests/overlay/087
> create mode 100644 tests/overlay/087.out
>
> diff --git a/tests/overlay/087 b/tests/overlay/087
> new file mode 100755
> index 00000000..100bb213
> --- /dev/null
> +++ b/tests/overlay/087
> @@ -0,0 +1,170 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (c) 2018 Red Hat, Inc. All Rights Reserved.
> +# Copyright (C) 2023 CTERA Networks. All Rights Reserved.
> +# Copyright (C) 2024 Mike Baynton. All Rights Reserved.
> +#
> +# FS QA Test 087
> +#
> +# Tests limited defined behaviors in case of additions to data-only layers
> +# while participating in a mounted overlayfs.
> +#
> +. ./common/preamble
> +_begin_fstest auto quick metacopy redirect
> +
> +# Import common functions.
> +. ./common/filter
> +. ./common/attr
> +
> +# real QA test starts here
> +_supported_fs overlay
> +# We use non-default scratch underlying overlay dirs, we need to check
> +# them explicity after test.
> +_require_scratch_nocheck
> +_require_scratch_overlay_features redirect_dir metacopy
> +_require_scratch_overlay_lowerdata_layers
> +_require_xfs_io_command "falloc"
> +
> +# remove all files from previous tests
> +_scratch_mkfs
> +
> +# File size on lower
> +dataname="d1/datafile"
> +datacontent="data"
> +dataname2="d2/datafile2"
> +datacontent2="data2"
> +datasize="4096"
> +
> +# Check size
> +check_file_size()
> +{
> + local target=$1 expected_size=$2 actual_size
> +
> + actual_size=$(_get_filesize $target)
> +
> + [ "$actual_size" == "$expected_size" ] || echo "Expected file size $expected_size but actual size is $actual_size"
> +}
> +
> +check_file_contents()
> +{
> + local target=$1 expected="$2"
> + local actual target_f
> +
> + target_f=`echo $target | _filter_scratch`
> +
> + read actual<"$target"
> +
> + [ "$actual" == "$expected" ] || echo "Expected file $target_f contents to be \"$expected\" but actual contents are \"$actual\""
> +}
> +
> +check_file_size_contents()
> +{
> + local target=$1 expected_size=$2 expected_content="$3"
> +
> + check_file_size $target $expected_size
> + check_file_contents $target "$expected_content"
> +}
> +
> +create_basic_files()
> +{
> + _scratch_mkfs
> + # create a few different directories on the data layer
> + mkdir -p "$datadir/d1" "$datadir/d2" "$lowerdir" "$upperdir" "$workdir"
> + echo "$datacontent" > $datadir/$dataname
> + chmod 600 $datadir/$dataname
> + echo "$datacontent2" > $datadir/$dataname2
> + chmod 600 $datadir/$dataname2
> +
> + # Create files of size datasize.
> + for f in $datadir/$dataname $datadir/$dataname2; do
> + $XFS_IO_PROG -c "falloc 0 $datasize" $f
> + $XFS_IO_PROG -c "fsync" $f
> + done
> +}
> +
> +mount_overlay()
> +{
> + _overlay_scratch_mount_opts \
> + -o"lowerdir=$lowerdir::$datadir" \
> + -o"upperdir=$upperdir,workdir=$workdir" \
> + -o redirect_dir=on,metacopy=on
> +}
> +
> +umount_overlay()
> +{
> + $UMOUNT_PROG $SCRATCH_MNT
> +}
> +
> +prepare_midlayer()
> +{
> + _scratch_mkfs
> + create_basic_files
> + # Create midlayer
> + _overlay_scratch_mount_dirs $datadir $lowerdir $workdir -o redirect_dir=on,index=on,metacopy=on
> + # Trigger metacopy and redirect xattrs
> + mv "$SCRATCH_MNT/$dataname" "$SCRATCH_MNT/file1"
> + mv "$SCRATCH_MNT/$dataname2" "$SCRATCH_MNT/file2"
> + umount_overlay
> +}
> +
> +# Create test directories
> +datadir=$OVL_BASE_SCRATCH_MNT/data
> +lowerdir=$OVL_BASE_SCRATCH_MNT/lower
> +upperdir=$OVL_BASE_SCRATCH_MNT/upper
> +workdir=$OVL_BASE_SCRATCH_MNT/workdir
> +
> +echo -e "\n== Create overlayfs and access files in data layer =="
> +#set -x
> +prepare_midlayer
> +mount_overlay
> +
> +# This creates a lookup under $datadir/d1, the directory later appended
> +check_file_size_contents "$SCRATCH_MNT/file1" $datasize $datacontent
> +# iterate some dirs through the overlayfs to populate caches
> +ls $SCRATCH_MNT > /dev/null
> +ls $SCRATCH_MNT/d1 > /dev/null
> +
> +echo -e "\n== Add new files to data layer, online and offline =="
> +
> +f="$OVL_BASE_SCRATCH_MNT/birthing_file"
> +echo "new file 1" > $f
> +chmod 600 $f
> +$XFS_IO_PROG -c "falloc 0 $datasize" $f
> +$XFS_IO_PROG -c "fsync" $f
> +# rename completed file under mounted ovl's data dir
> +mv $f $datadir/d1/newfile1
> +
> +newfile1="$SCRATCH_MNT/d1/newfile1"
> +newfile2="$SCRATCH_MNT/d1/newfile2"
> +# Try to open some files that will exist in future
> +read <"$newfile1" 2>/dev/null || echo "newfile1 expected missing"
> +read <"$newfile2" 2>/dev/null || echo "newfile2 expected missing"
This does not cause the bash ENOENT failure
message redirect to /dev/null
suggest to use either:
cat "$newfile2" &>/dev/null
or simply:
test -f "$newfile2" && echo "newfile2 expected missing"
> +
> +umount_overlay
> +
> +echo "new file 2" > "$datadir/d1/newfile2"
> +chmod 600 "$datadir/d1/newfile2"
> +$XFS_IO_PROG -c "falloc 0 $datasize" "$datadir/d1/newfile2"
> +$XFS_IO_PROG -c "fsync" "$datadir/d1/newfile2"
> +
> +# Add new files to midlayer with redirects to the files we appended to the lower dir
> +_overlay_scratch_mount_dirs $datadir $lowerdir $workdir -o redirect_dir=on,index=on,metacopy=on
> +mv "$newfile1" "$SCRATCH_MNT/_newfile1"
> +mv "$newfile2" "$SCRATCH_MNT/_newfile2"
> +umount_overlay
> +mv "$lowerdir/_newfile1" "$lowerdir/d1/newfile1"
> +mv "$lowerdir/_newfile2" "$lowerdir/d1/newfile2"
This offline rename is not needed - you can do it online.
There is no reason for this test to endorse offline changes like this
which are not part of the two behaviors that you had set to formalize.
> +
> +echo -e "\n== Verify files appended to data layer while mounted are available after remount =="
> +mount_overlay
> +
> +ls "$SCRATCH_MNT/d1"
> +check_file_size_contents "$newfile1" $datasize "new file 1"
> +check_file_size_contents "$newfile2" $datasize "new file 2"
> +check_file_size_contents "$SCRATCH_MNT/file1" $datasize $datacontent
> +
> +umount_overlay
> +
> +# success, all done
> +status=0
> +exit
> diff --git a/tests/overlay/087.out b/tests/overlay/087.out
> new file mode 100644
> index 00000000..db16c8a2
> --- /dev/null
> +++ b/tests/overlay/087.out
> @@ -0,0 +1,13 @@
> +QA output created by 087
> +
> +== Create overlayfs and access files in data layer ==
> +
> +== Add new files to data layer, online and offline ==
> +/root/projects/xfstests-dev/tests/overlay/087: line 138: /mnt/scratch/ovl-mnt/d1/newfile1: No such file or directory
> +newfile1 expected missing
> +/root/projects/xfstests-dev/tests/overlay/087: line 139: /mnt/scratch/ovl-mnt/d1/newfile2: No such file or directory
Those errors should not be in the "golden output"
because they will not match the exact string on any test env.
The generic solution is to use _filter_* helpers to canonicalize the
golden output
but in this case, the stderr text is not needed at all and should be
redirected to /dev/null
or avoided (see above)
> +newfile2 expected missing
> +
> +== Verify files appended to data layer while mounted are available after remount ==
> +newfile1
> +newfile2
> --
Thanks,
Amir.
