Checking hashes for files is a handy utility function for
client side tests, so moving the function hash_file to
autotest_lib.client.bin.utils, and updating the function
unmap_url_cache. Also updated the parts of the code that
reference the previously mentioned functions.
Signed-off-by: Lucas Meneghel Rodrigues <lmr@xxxxxxxxxx>
---
client/bin/base_utils.py | 91 ++++++++++++++++++++++++++++++++---------
client/tests/kvm/cd_hash.py | 17 ++++----
client/tests/kvm/kvm_utils.py | 88 ---------------------------------------
client/tests/kvm/kvm_vm.py | 7 ++-
4 files changed, 83 insertions(+), 120 deletions(-)
diff --git a/client/bin/base_utils.py b/client/bin/base_utils.py
index b4f5646..4c1830e 100644
--- a/client/bin/base_utils.py
+++ b/client/bin/base_utils.py
@@ -94,41 +94,92 @@ def extract_tarball(tarball):
raise NameError('extracting tarball produced no dir')
-def get_md5sum(file_path):
- """Gets the md5sum of a file. You must provide a valid path to the file"""
- if not os.path.isfile(file_path):
- raise ValueError, 'invalid file %s to verify' % file_path
- md5sum = utils.system_output("md5sum " + file_path)
- return md5sum.split()[0]
+def hash_file(filename, size=None, method="md5"):
+ """
+ Calculate the hash of filename.
+ If size is not None, limit to first size bytes.
+ Throw exception if something is wrong with filename.
+ Can be also implemented with bash one-liner (assuming size%1024==0):
+ dd if=filename bs=1024 count=size/1024 | sha1sum -
+
+ @param filename: Path of the file that will have its hash calculated.
+ @param method: Method used to calculate the hash. Supported methods:
+ * md5
+ * sha1
+ @returns: Hash of the file, if something goes wrong, return None.
+ """
+ chunksize = 4096
+ fsize = os.path.getsize(filename)
+
+ if not size or size > fsize:
+ size = fsize
+ f = open(filename, 'rb')
+
+ hash = utils.hash(method)
+
+ while size > 0:
+ if chunksize > size:
+ chunksize = size
+ data = f.read(chunksize)
+ if len(data) == 0:
+ logging.debug("Nothing left to read but size=%d" % size)
+ break
+ hash.update(data)
+ size -= len(data)
+ f.close()
+ return hash.hexdigest()
-def unmap_url_cache(cachedir, url, expected_md5):
+def unmap_url_cache(cachedir, url, expected_hash, method="md5"):
"""
Downloads a file from a URL to a cache directory. If the file is already
- at the expected position and has the expected md5 number, let's not
- download it again.
+ at the expected position and has the expected hash, let's not download it
+ again.
+
+ @param cachedir: Directory that might hold a copy of the file we want to
+ download.
+ @param url: URL for the file we want to download.
+ @param expected_hash: Hash string that we expect the file downloaded to
+ have.
+ @param method: Method used to calculate the hash string (md5, sha1).
"""
# Let's convert cachedir to a canonical path, if it's not already
cachedir = os.path.realpath(cachedir)
if not os.path.isdir(cachedir):
try:
- utils.system('mkdir -p ' + cachedir)
+ os.makedirs(cachedir)
except:
raise ValueError('Could not create cache directory %s' % cachedir)
file_from_url = os.path.basename(url)
file_local_path = os.path.join(cachedir, file_from_url)
- if os.path.isfile(file_local_path):
- file_md5 = get_md5sum(file_local_path)
- if file_md5 == expected_md5:
- # File is already at the expected position and ready to go
- src = file_from_url
+
+ file_hash = None
+ failure_counter = 0
+ while not file_hash == expected_hash:
+ if os.path.isfile(file_local_path):
+ file_hash = hash_file(file_local_path, method)
+ if file_hash == expected_hash:
+ # File is already at the expected position and ready to go
+ src = file_from_url
+ else:
+ # Let's download the package again, it's corrupted...
+ logging.error("Seems that file %s is corrupted, trying to "
+ "download it again" % file_from_url)
+ src = url
+ failure_counter += 1
else:
- # Let's download the package again, it's corrupted...
+ # File is not there, let's download it
src = url
- else:
- # File is not there, let's download it
- src = url
- return utils.unmap_url(cachedir, src, cachedir)
+ if failure_counter > 1:
+ raise EnvironmentError("Consistently failed to download the "
+ "package %s. Aborting further download "
+ "attempts. This might mean either the "
+ "network connection has problems or the "
+ "expected hash string that was determined "
+ "for this file is wrong" % file_from_url)
+ file_path = utils.unmap_url(cachedir, src, cachedir)
+
+ return file_path
def force_copy(src, dest):
diff --git a/client/tests/kvm/cd_hash.py b/client/tests/kvm/cd_hash.py
index 483d71c..b8a03bf 100755
--- a/client/tests/kvm/cd_hash.py
+++ b/client/tests/kvm/cd_hash.py
@@ -6,8 +6,9 @@ Program that calculates several hashes for a given CD image.
"""
import os, sys, optparse, logging
-import common, kvm_utils
+import common
from autotest_lib.client.common_lib import logging_config, logging_manager
+from autotest_lib.client.bin import utils
class KvmLoggingConfig(logging_config.LoggingConfig):
@@ -44,11 +45,9 @@ if __name__ == "__main__":
sys.exit(1)
logging.info("Hash values for file %s", os.path.basename(filename))
- logging.info("md5 (1m): %s", kvm_utils.hash_file(filename, 1024*1024,
- method="md5"))
- logging.info("sha1 (1m): %s", kvm_utils.hash_file(filename, 1024*1024,
- method="sha1"))
- logging.info("md5 (full): %s", kvm_utils.hash_file(filename,
- method="md5"))
- logging.info("sha1 (full): %s", kvm_utils.hash_file(filename,
- method="sha1"))
+ logging.info("md5 (1m): %s", utils.hash_file(filename, 1024*1024,
+ method="md5"))
+ logging.info("sha1 (1m): %s", utils.hash_file(filename, 1024*1024,
+ method="sha1"))
+ logging.info("md5 (full): %s", utils.hash_file(filename, method="md5"))
+ logging.info("sha1 (full): %s", utils.hash_file(filename, method="sha1"))
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index da7c543..cc2f678 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -767,42 +767,6 @@ def wait_for(func, timeout, first=0.0, step=1.0, text=None):
return None
-def hash_file(filename, size=None, method="md5"):
- """
- Calculate the hash of filename.
- If size is not None, limit to first size bytes.
- Throw exception if something is wrong with filename.
- Can be also implemented with bash one-liner (assuming size%1024==0):
- dd if=filename bs=1024 count=size/1024 | sha1sum -
-
- @param filename: Path of the file that will have its hash calculated.
- @param method: Method used to calculate the hash. Supported methods:
- * md5
- * sha1
- @returns: Hash of the file, if something goes wrong, return None.
- """
- chunksize = 4096
- fsize = os.path.getsize(filename)
-
- if not size or size > fsize:
- size = fsize
- f = open(filename, 'rb')
-
- hash = utils.hash(method)
-
- while size > 0:
- if chunksize > size:
- chunksize = size
- data = f.read(chunksize)
- if len(data) == 0:
- logging.debug("Nothing left to read but size=%d" % size)
- break
- hash.update(data)
- size -= len(data)
- f.close()
- return hash.hexdigest()
-
-
def get_hash_from_file(hash_path, dvd_basename):
"""
Get the a hash from a given DVD image from a hash file
@@ -818,58 +782,6 @@ def get_hash_from_file(hash_path, dvd_basename):
return line.split()[0]
-def unmap_url_cache(cachedir, url, expected_hash, method="md5"):
- """
- Downloads a file from a URL to a cache directory. If the file is already
- at the expected position and has the expected hash, let's not download it
- again.
-
- @param cachedir: Directory that might hold a copy of the file we want to
- download.
- @param url: URL for the file we want to download.
- @param expected_hash: Hash string that we expect the file downloaded to
- have.
- @param method: Method used to calculate the hash string (md5, sha1).
- """
- # Let's convert cachedir to a canonical path, if it's not already
- cachedir = os.path.realpath(cachedir)
- if not os.path.isdir(cachedir):
- try:
- os.makedirs(cachedir)
- except:
- raise ValueError('Could not create cache directory %s' % cachedir)
- file_from_url = os.path.basename(url)
- file_local_path = os.path.join(cachedir, file_from_url)
-
- file_hash = None
- failure_counter = 0
- while not file_hash == expected_hash:
- if os.path.isfile(file_local_path):
- file_hash = hash_file(file_local_path, method)
- if file_hash == expected_hash:
- # File is already at the expected position and ready to go
- src = file_from_url
- else:
- # Let's download the package again, it's corrupted...
- logging.error("Seems that file %s is corrupted, trying to "
- "download it again" % file_from_url)
- src = url
- failure_counter += 1
- else:
- # File is not there, let's download it
- src = url
- if failure_counter > 1:
- raise EnvironmentError("Consistently failed to download the "
- "package %s. Aborting further download "
- "attempts. This might mean either the "
- "network connection has problems or the "
- "expected hash string that was determined "
- "for this file is wrong" % file_from_url)
- file_path = utils.unmap_url(cachedir, src, cachedir)
-
- return file_path
-
-
def run_tests(test_list, job):
"""
Runs the sequence of KVM tests based on the list of dictionaries
diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index 6731927..eb61ba2 100755
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -7,6 +7,7 @@ Utility classes and functions to handle Virtual Machine creation using qemu.
import time, socket, os, logging, fcntl, re, commands
import kvm_utils, kvm_subprocess
+from autotest_lib.client.bin import utils
def get_image_filename(params, root_dir):
@@ -362,19 +363,19 @@ class VM:
if params.get("md5sum_1m"):
logging.debug("Comparing expected MD5 sum with MD5 sum of "
"first MB of ISO file...")
- actual_hash = kvm_utils.hash_file(iso, 1048576, method="md5")
+ actual_hash = utils.hash_file(iso, 1048576, method="md5")
expected_hash = params.get("md5sum_1m")
compare = True
elif params.get("md5sum"):
logging.debug("Comparing expected MD5 sum with MD5 sum of ISO "
"file...")
- actual_hash = kvm_utils.hash_file(iso, method="md5")
+ actual_hash = utils.hash_file(iso, method="md5")
expected_hash = params.get("md5sum")
compare = True
elif params.get("sha1sum"):
logging.debug("Comparing expected SHA1 sum with SHA1 sum of "
"ISO file...")
- actual_hash = kvm_utils.hash_file(iso, method="sha1")
+ actual_hash = utils.hash_file(iso, method="sha1")
expected_hash = params.get("sha1sum")
compare = True
if compare:
--
1.6.6
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
