On 12/4/18 6:47 PM, Josef Bacik wrote:
> I wrote these scripts for xfstests, just copying them over to blktests
> as well. The terse output fio support that blktests doesn't get all of
> the various fio performance things that we may want to look at, this
> gives us a lot of flexibility around getting specific values out of the
> fio results data.
>
Hi,
some suggestions below :)
> Signed-off-by: Josef Bacik <josef@xxxxxxxxxxxxxx>
> ---
> src/FioResultDecoder.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++
> src/fio-key-value.py | 28 ++++++++++++++++++++++
> 2 files changed, 92 insertions(+)
> create mode 100644 src/FioResultDecoder.py
> create mode 100644 src/fio-key-value.py
>
> diff --git a/src/FioResultDecoder.py b/src/FioResultDecoder.py
> new file mode 100644
> index 000000000000..d004140c0fdf
> --- /dev/null
> +++ b/src/FioResultDecoder.py
> @@ -0,0 +1,64 @@
I think a shebang could clarify this runs under python2.
> +# SPDX-License-Identifier: GPL-2.0
> +
> +import json
> +
> +class FioResultDecoder(json.JSONDecoder):
> + """Decoder for decoding fio result json to an object for our database
> +
> + This decodes the json output from fio into an object that can be directly
> + inserted into our database. This just strips out the fields we don't care
> + about and collapses the read/write/trim classes into a flat value structure
> + inside of the jobs object.
> +
> + For example
> + "write" : {
> + "io_bytes" : 313360384,
> + "bw" : 1016,
> + }
> +
> + Get's collapsed to
> +
> + "write_io_bytes" : 313360384,
> + "write_bw": 1016,
> +
> + Currently any dict under 'jobs' get's dropped, with the exception of 'read',
> + 'write', and 'trim'. For those sub sections we drop any dict's under those.
> +
> + Attempt to keep this as generic as possible, we don't want to break every
> + time fio changes it's json output format.
> + """
> + _ignore_types = ['dict', 'list']
> + _override_keys = ['lat_ns', 'lat']
> + _io_ops = ['read', 'write', 'trim']
> +
> + _transform_keys = { 'lat': 'lat_ns' }
> +
> + def decode(self, json_string):
> + """This does the dirty work of converting everything"""
> + default_obj = super(FioResultDecoder, self).decode(json_string)
> + obj = {}
> + obj['global'] = {}
> + obj['global']['time'] = default_obj['time']
> + obj['jobs'] = []
> + for job in default_obj['jobs']:
> + new_job = {}
> + for key,value in job.iteritems():
> + if key not in self._io_ops:
> + if value.__class__.__name__ in self._ignore_types:
> + continue
> + new_job[key] = value
> + continue
> + for k,v in value.iteritems():
> + if k in self._override_keys:
> + if k in self._transform_keys:
> + k = self._transform_keys[k]
> + for subk,subv in v.iteritems():
> + collapsed_key = "{}_{}_{}".format(key, k, subk)
> + new_job[collapsed_key] = subv
> + continue
> + if v.__class__.__name__ in self._ignore_types:
> + continue
> + collapsed_key = "{}_{}".format(key, k)
> + new_job[collapsed_key] = v
> + obj['jobs'].append(new_job)
> + return obj
> diff --git a/src/fio-key-value.py b/src/fio-key-value.py
> new file mode 100644
> index 000000000000..208e9a453a19
> --- /dev/null
> +++ b/src/fio-key-value.py
> @@ -0,0 +1,28 @@
Also here a shebang could be fine.
> +# SPDX-License-Identifier: GPL-2.0
> +
> +import FioResultDecoder
> +import json
> +import argparse
> +import sys
> +import platform
> +
> +parser = argparse.ArgumentParser()
> +parser.add_argument('-j', '--jobname', type=str,
> + help="The jobname we want our key from.",
> + required=True)
> +parser.add_argument('-k', '--key', type=str,
> + help="The key we want the value of", required=True)
> +parser.add_argument('result', type=str,
If here you set type=open ^
> + help="The result file read")
> +args = parser.parse_args()
> +
> +json_data = open(args.result)
you could avoid this line ^
> +data = json.load(json_data, cls=FioResultDecoder.FioResultDecoder)
and substitute json_data with args.result which would return a file object.
In short the above piece of patch would become something like:
parser.add_argument('result', type=open, help="The result file read")
args = parser.parse_args()
data = json.load(args.result, cls=FioResultDecoder.FioResultDecoder)
Which still raises IOError if bad arguments are passed.
> +
> +for job in data['jobs']:
> + if job['jobname'] == args.jobname:
> + if args.key not in job:
> + print('')
> + else:
> + print("{}".format(job[args.key]))
> + break
>
Have a nice day,
Federico
Attachment:
signature.asc
Description: OpenPGP digital signature
