RE: [PATCH v1 1/1] engines: add engine for file delete

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hi,

Thanks for your comments.

>> Can't delete files from stdout/stdin?
We expect fio auto create batch of files, then filedelete engine deletes them.
Create from stdin/out seems not so meaningful.
Meanwhile, if set 'filename=-', generic_get_file_size() can not get actual size.
I referred to engines/filecreate.c.

Other comments, I will update patch.

-----Original Message-----
From: Sitsofe Wheeler <sitsofe@xxxxxxxxx> 
Sent: Thursday, February 25, 2021 3:21 PM
To: Su, Friendy <Friendy.Su@xxxxxxxx>
Cc: fio <fio@xxxxxxxxxxxxxxx>; Kobayashi, Kento (Sony) <Kento.A.Kobayashi@xxxxxxxx>; Aoyama, Wataru (Sony) <Wataru.Aoyama@xxxxxxxx>
Subject: Re: [PATCH v1 1/1] engines: add engine for file delete

Hi,

On Mon, 22 Feb 2021 at 12:15, <Friendy.Su@xxxxxxxx> wrote:
>
> This engine is to measure performance of deleting files.
>
> In practice, it is an important benchmark for a file system that how 
> quick it can release the disk space of deleted files.
>
> Signed-off-by: friendy-su <friendy.su@xxxxxxxx>
> ---
>  HOWTO                            |   5 ++
>  Makefile                         |   2 +-
>  engines/filedelete.c             | 108 +++++++++++++++++++++++++++++++
>  examples/filedelete-ioengine.fio |  23 +++++++
>  fio.1                            |   5 ++
>  5 files changed, 142 insertions(+), 1 deletion(-)  create mode 100644 
> engines/filedelete.c  create mode 100644 
> examples/filedelete-ioengine.fio
>
> diff --git a/HOWTO b/HOWTO
> index b6d1b58a..ea728439 100644
> --- a/HOWTO
> +++ b/HOWTO
> @@ -2033,6 +2033,11 @@ I/O engine
>                         and 'nrfiles', so that files will be created.
>                         This engine is to measure file lookup and meta data access.
>
> +               **filedelete**
> +                       Simply delete the files by unlink() and do no I/O to them. You need to set 'filesize'
> +                       and 'nrfiles', so that the files will be created.
> +                       This engine is to measure file delete.
> +
>                 **libpmem**
>                         Read and write using mmap I/O to a file on a filesystem
>                         mounted with DAX on a persistent memory device 
> through the PMDK diff --git a/Makefile b/Makefile index 
> 612344d1..11ffa089 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -51,7 +51,7 @@ SOURCE :=     $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
>                 pshared.c options.c \
>                 smalloc.c filehash.c profile.c debug.c engines/cpu.c \
>                 engines/mmap.c engines/sync.c engines/null.c engines/net.c \
> -               engines/ftruncate.c engines/filecreate.c engines/filestat.c \
> +               engines/ftruncate.c engines/filecreate.c 
> + engines/filestat.c engines/filedelete.c \
>                 server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \
>                 gettime-thread.c helpers.c json.c idletime.c td_error.c \
>                 profiles/tiobench.c profiles/act.c io_u_queue.c 
> filelock.c \ diff --git a/engines/filedelete.c b/engines/filedelete.c 
> new file mode 100644 index 00000000..71977b22
> --- /dev/null
> +++ b/engines/filedelete.c
> @@ -0,0 +1,108 @@
> +/*
> + * file delete engine
> + *
> + * IO engine that doesn't do any IO, just delete files and tracks the 
> +latency
> + * of the file deletion.
> + */
> +#include <stdio.h>
> +#include <fcntl.h>
> +#include <errno.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include "../fio.h"
> +
> +struct fc_data {
> +       enum fio_ddir stat_ddir;
> +};
> +
> +static int delete_file(struct thread_data *td, struct fio_file *f) {
> +       struct timespec start;
> +       int do_lat = !td->o.disable_lat;
> +       int ret;
> +
> +       dprint(FD_FILE, "fd delete %s\n", f->file_name);
> +
> +       if (f->filetype != FIO_TYPE_FILE) {
> +               log_err("fio: only files are supported fallocate \n");

This message doesn't quite make sense... only files are supported by filedelete?

> +               return 1;
> +       }
> +       if (!strcmp(f->file_name, "-")) {
> +               log_err("fio: can't read/write to stdin/out\n");

Can't delete files from stdout/stdin?

> +               return 1;
> +       }
> +
> +       if (do_lat)
> +               fio_gettime(&start, NULL);
> +
> +       ret = unlink(f->file_name);
> +
> +       if (ret == -1) {
> +               char buf[FIO_VERROR_SIZE];
> +               int e = errno;
> +
> +               snprintf(buf, sizeof(buf), "delete(%s)", f->file_name);
> +               td_verror(td, e, buf);
> +               return 1;
> +       }
> +
> +       if (do_lat) {
> +               struct fc_data *data = td->io_ops_data;
> +               uint64_t nsec;
> +
> +               nsec = ntime_since_now(&start);
> +               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0);
> +       }
> +
> +       return 0;
> +}
> +
> +
> +static enum fio_q_status queue_io(struct thread_data *td, struct io_u 
> +fio_unused *io_u) {
> +       return FIO_Q_COMPLETED;
> +}
> +
> +static int init(struct thread_data *td) {
> +       struct fc_data *data;
> +
> +       data = calloc(1, sizeof(*data));
> +
> +       if (td_read(td))
> +               data->stat_ddir = DDIR_READ;
> +       else if (td_write(td))
> +               data->stat_ddir = DDIR_WRITE;
> +
> +       td->io_ops_data = data;
> +       return 0;
> +}
> +
> +static void cleanup(struct thread_data *td) {
> +       struct fc_data *data = td->io_ops_data;
> +
> +       free(data);
> +}
> +
> +static struct ioengine_ops ioengine = {
> +       .name           = "filedelete",
> +       .version        = FIO_IOOPS_VERSION,
> +       .init           = init,
> +       .cleanup        = cleanup,
> +       .queue          = queue_io,
> +       .get_file_size  = generic_get_file_size,
> +       .open_file      = delete_file,
> +       .flags          =  FIO_SYNCIO | FIO_FAKEIO |
> +                               FIO_NOSTATS | FIO_NOFILEHASH, };
> +
> +static void fio_init fio_filecreate_register(void) {
> +       register_ioengine(&ioengine);
> +}
> +
> +static void fio_exit fio_filecreate_unregister(void) {
> +       unregister_ioengine(&ioengine); }
> diff --git a/examples/filedelete-ioengine.fio 
> b/examples/filedelete-ioengine.fio
> new file mode 100644
> index 00000000..58002c8a
> --- /dev/null
> +++ b/examples/filedelete-ioengine.fio
> @@ -0,0 +1,23 @@
> +# Example filedelete job
> +
> +# 'filedelete' engine only do 'unlink(filename)', file will not be open().
> +# 'invalidate' should be set to 0 since file will not be open(), posix_fadvise(fd,....) will fail.

Did you want the ioengine to enforce this? If so you can "implement"
the .invalidate op and simply do nothing (see engines/filestat.c:173).
Others catch settings in init (e.g. engines/mmap.c:272).

> +# 'filesize' must be set, then files will be created at setup stage.
> +# 'unlink' must be set to 0, since the file is deleted in measurement.
> +
> +[global]
> +ioengine=filedelete
> +numjobs=1
> +filesize=4k
> +invalidate=0
> +nrfiles=200
> +unlink=0
> +
> +[t0]
> +[t1]
> +[t2]
> +[t3]
> +[t4]
> +[t5]
> +
> +

You can probably trim the trailing empty lines.

> diff --git a/fio.1 b/fio.1
> index aa248a3b..4b1ef80d 100644
> --- a/fio.1
> +++ b/fio.1
> @@ -1819,6 +1819,11 @@ Simply do stat() and do no I/O to the file. You need to set 'filesize'
>  and 'nrfiles', so that files will be created.
>  This engine is to measure file lookup and meta data access.
>  .TP
> +.B filestat
> +Simply delete files by unlink() and do no I/O to the file. You need to set 'filesize'
> +and 'nrfiles', so that files will be created.
> +This engine is to measure file delete.
> +.TP

This looks different to what you put in the HOWTO. Is this intentional?

>  .B libpmem
>  Read and write using mmap I/O to a file on a filesystem  mounted with 
> DAX on a persistent memory device through the PMDK
> --
> 2.17.1
>


--
Sitsofe




[Index of Archives]     [Linux Kernel]     [Linux SCSI]     [Linux IDE]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux SCSI]

  Powered by Linux