From: Eren Yagdiran <erenyagdiran@xxxxxxxxx>
Refactor download function from virt-sandbox-image to use
the newly introduced Source abstract class. The docker-specific
download code is moved to a new DockerSource class.
Signed-off-by: Daniel P. Berrange <berrange@xxxxxxxxxx>
---
libvirt-sandbox/image/cli.py | 204 ++++---------------------
libvirt-sandbox/image/sources/DockerSource.py | 209 ++++++++++++++++++++++++++
libvirt-sandbox/image/sources/Makefile.am | 1 +
libvirt-sandbox/image/sources/Source.py | 15 ++
4 files changed, 257 insertions(+), 172 deletions(-)
create mode 100644 libvirt-sandbox/image/sources/DockerSource.py
diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py
index de34321..7af617e 100755
--- a/libvirt-sandbox/image/cli.py
+++ b/libvirt-sandbox/image/cli.py
@@ -69,176 +69,6 @@ def debug(msg):
def info(msg):
sys.stdout.write(msg)
-def get_url(server, path, headers):
- url = "https://"; + server + path
- debug(" Fetching %s..." % url)
- req = urllib2.Request(url=url)
-
- if json:
- req.add_header("Accept", "application/json")
-
- for h in headers.keys():
- req.add_header(h, headers[h])
-
- return urllib2.urlopen(req)
-
-def get_json(server, path, headers):
- try:
- res = get_url(server, path, headers)
- data = json.loads(res.read())
- debug("OK\n")
- return (data, res)
- except Exception, e:
- debug("FAIL %s\n" % str(e))
- raise
-
-def save_data(server, path, headers, dest, checksum=None, datalen=None):
- try:
- res = get_url(server, path, headers)
-
- csum = None
- if checksum is not None:
- csum = hashlib.sha256()
-
- pattern = [".", "o", "O", "o"]
- patternIndex = 0
- donelen = 0
-
- with open(dest, "w") as f:
- while 1:
- buf = res.read(1024*64)
- if not buf:
- break
- if csum is not None:
- csum.update(buf)
- f.write(buf)
-
- if datalen is not None:
- donelen = donelen + len(buf)
- debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
- pattern[patternIndex], (donelen/1024), (datalen/1024)
- ))
- patternIndex = (patternIndex + 1) % 4
-
- debug("\x1b[K")
- if csum is not None:
- csumstr = "sha256:" + csum.hexdigest()
- if csumstr != checksum:
- debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
- os.remove(dest)
- raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
- debug("OK\n")
- return res
- except Exception, e:
- debug("FAIL %s\n" % str(e))
- raise
-
-
-def download_template(name, server, destdir):
- tag = "latest"
-
- offset = name.find(':')
- if offset != -1:
- tag = name[offset + 1:]
- name = name[0:offset]
-
- # First we must ask the index server about the image name. THe
- # index server will return an auth token we can use when talking
- # to the registry server. We need this token even when anonymous
- try:
- (data, res) = get_json(server, "/v1/repositories/" + name + "/images",
- {"X-Docker-Token": "true"})
- except urllib2.HTTPError, e:
- raise ValueError(["Image '%s' does not exist" % name])
-
- registryserver = res.info().getheader('X-Docker-Endpoints')
- token = res.info().getheader('X-Docker-Token')
- checksums = {}
- for layer in data:
- pass
- # XXX Checksums here don't appear to match the data in
- # image download later. Find out what these are sums of
- #checksums[layer["id"]] = layer["checksum"]
-
- # Now we ask the registry server for the list of tags associated
- # with the image. Tags usually reflect some kind of version of
- # the image, but they aren't officially "versions". There is
- # always a "latest" tag which is the most recent upload
- #
- # We must pass in the auth token from the index server. This
- # token can only be used once, and we're given a cookie back
- # in return to use for later RPC calls.
- (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags",
- { "Authorization": "Token " + token })
-
- cookie = res.info().getheader('Set-Cookie')
-
- if not tag in data:
- raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)])
- imagetagid = data[tag]
-
- # Only base images are self-contained, most images reference one
- # or more parents, in a linear stack. Here we are getting the list
- # of layers for the image with the tag we used.
- (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry",
- { "Authorization": "Token " + token })
-
- if data[0] != imagetagid:
- raise ValueError(["Expected first layer id '%s' to match image id '%s'",
- data[0], imagetagid])
-
- try:
- createdFiles = []
- createdDirs = []
-
- for layerid in data:
- templatedir = destdir + "/" + layerid
- if not os.path.exists(templatedir):
- os.mkdir(templatedir)
- createdDirs.append(templatedir)
-
- jsonfile = templatedir + "/template.json"
- datafile = templatedir + "/template.tar.gz"
-
- if not os.path.exists(jsonfile) or not os.path.exists(datafile):
- # The '/json' URL gives us some metadata about the layer
- res = save_data(registryserver, "/v1/images/" + layerid + "/json",
- { "Authorization": "Token " + token }, jsonfile)
- createdFiles.append(jsonfile)
- layersize = int(res.info().getheader("Content-Length"))
-
- datacsum = None
- if layerid in checksums:
- datacsum = checksums[layerid]
-
- # and the '/layer' URL is the actual payload, provided
- # as a tar.gz archive
- save_data(registryserver, "/v1/images/" + layerid + "/layer",
- { "Authorization": "Token " + token }, datafile, datacsum, layersize)
- createdFiles.append(datafile)
-
- # Strangely the 'json' data for a layer doesn't include
- # its actual name, so we save that in a json file of our own
- index = {
- "name": name,
- }
-
- indexfile = destdir + "/" + imagetagid + "/index.json"
- with open(indexfile, "w") as f:
- f.write(json.dumps(index))
- except Exception, e:
- for f in createdFiles:
- try:
- os.remove(f)
- except:
- pass
- for d in createdDirs:
- try:
- os.rmdir(d)
- except:
- pass
-
-
def delete_template(name, destdir):
imageusage = {}
imageparent = {}
@@ -342,8 +172,16 @@ def create_template(name, imagepath, format, destdir):
parentImage = templateImage
def download(args):
- info("Downloading %s from %s to %s\n" % (args.template, default_index_server, default_template_dir))
- download_template(args.template, default_index_server, default_template_dir)
+ try:
+ dynamic_source_loader(args.source).download_template(templatename=args.template,
+ templatedir=args.template_dir,
+ registry=args.registry,
+ username=args.username,
+ password=args.password)
+ except IOError,e:
+ print "Source %s cannot be found in given path" %args.source
+ except Exception,e:
+ print "Download Error %s" % str(e)
def delete(args):
info("Deleting %s from %s\n" % (args.template, default_template_dir))
@@ -357,10 +195,32 @@ def requires_template(parser):
parser.add_argument("template",
help=_("name of the template"))
+def requires_source(parser):
+ parser.add_argument("-s","--source",
+ default="docker",
+ help=_("name of the template"))
+
+def requires_auth_conn(parser):
+ parser.add_argument("-r","--registry",
+ help=_("Url of the custom registry"))
+ parser.add_argument("-u","--username",
+ help=_("Username for the custom registry"))
+ parser.add_argument("-p","--password",
+ help=_("Password for the custom registry"))
+
+def requires_template_dir(parser):
+ global default_template_dir
+ parser.add_argument("-t","--template-dir",
+ default=default_template_dir,
+ help=_("Template directory for saving templates"))
+
def gen_download_args(subparser):
parser = subparser.add_parser("download",
help=_("Download template data"))
requires_template(parser)
+ requires_source(parser)
+ requires_auth_conn(parser)
+ requires_template_dir(parser)
parser.set_defaults(func=download)
def gen_delete_args(subparser):
diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py
new file mode 100644
index 0000000..37b40dc
--- /dev/null
+++ b/libvirt-sandbox/image/sources/DockerSource.py
@@ -0,0 +1,209 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 Universitat Politècnica de Catalunya.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Author: Eren Yagdiran <erenyagdiran@xxxxxxxxx>
+#
+
+from Source import Source
+import urllib2
+import sys
+import json
+import traceback
+import os
+import subprocess
+import shutil
+
+class DockerSource(Source):
+
+ www_auth_username = None
+ www_auth_password = None
+
+ def __init__(self):
+ self.default_index_server = "index.docker.io"
+
+ def _check_cert_validate(self):
+ major = sys.version_info.major
+ SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
+ SSL_WARNING +="\nSee https://bugs.python.org/issue22417\n";
+ py2_7_9_hexversion = 34015728
+ py3_4_3_hexversion = 50594800
+ if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion):
+ sys.stderr.write(SSL_WARNING)
+
+ def download_template(self, templatename, templatedir,
+ registry=None, username=None, password=None):
+ if registry is None:
+ registry = self.default_index_server
+
+ if username is not None:
+ self.www_auth_username = username
+ self.www_auth_password = password
+
+ self._check_cert_validate()
+ tag = "latest"
+ offset = templatename.find(':')
+ if offset != -1:
+ tag = templatename[offset + 1:]
+ templatename = templatename[0:offset]
+ try:
+ (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images",
+ {"X-Docker-Token": "true"})
+ except urllib2.HTTPError, e:
+ raise ValueError(["Image '%s' does not exist" % templatename])
+
+ registryendpoint = res.info().getheader('X-Docker-Endpoints')
+ token = res.info().getheader('X-Docker-Token')
+ checksums = {}
+ for layer in data:
+ pass
+ (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags",
+ { "Authorization": "Token " + token })
+
+ cookie = res.info().getheader('Set-Cookie')
+
+ if not tag in data:
+ raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)])
+ imagetagid = data[tag]
+
+ (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry",
+ { "Authorization": "Token "+token })
+
+ if data[0] != imagetagid:
+ raise ValueError(["Expected first layer id '%s' to match image id '%s'",
+ data[0], imagetagid])
+
+ try:
+ createdFiles = []
+ createdDirs = []
+
+ for layerid in data:
+ layerdir = templatedir + "/" + layerid
+ if not os.path.exists(layerdir):
+ os.makedirs(layerdir)
+ createdDirs.append(layerdir)
+
+ jsonfile = layerdir + "/template.json"
+ datafile = layerdir + "/template.tar.gz"
+
+ if not os.path.exists(jsonfile) or not os.path.exists(datafile):
+ res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json",
+ { "Authorization": "Token " + token }, jsonfile)
+ createdFiles.append(jsonfile)
+
+ layersize = int(res.info().getheader("Content-Length"))
+
+ datacsum = None
+ if layerid in checksums:
+ datacsum = checksums[layerid]
+
+ self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer",
+ { "Authorization": "Token "+token }, datafile, datacsum, layersize)
+ createdFiles.append(datafile)
+
+ index = {
+ "name": templatename,
+ }
+
+ indexfile = templatedir + "/" + imagetagid + "/index.json"
+ print("Index file " + indexfile)
+ with open(indexfile, "w") as f:
+ f.write(json.dumps(index))
+ except Exception as e:
+ traceback.print_exc()
+ for f in createdFiles:
+ try:
+ os.remove(f)
+ except:
+ pass
+ for d in createdDirs:
+ try:
+ shutil.rmtree(d)
+ except:
+ pass
+ def _save_data(self,server, path, headers, dest, checksum=None, datalen=None):
+ try:
+ res = self._get_url(server, path, headers)
+
+ csum = None
+ if checksum is not None:
+ csum = hashlib.sha256()
+
+ pattern = [".", "o", "O", "o"]
+ patternIndex = 0
+ donelen = 0
+
+ with open(dest, "w") as f:
+ while 1:
+ buf = res.read(1024*64)
+ if not buf:
+ break
+ if csum is not None:
+ csum.update(buf)
+ f.write(buf)
+
+ if datalen is not None:
+ donelen = donelen + len(buf)
+ debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
+ pattern[patternIndex], (donelen/1024), (datalen/1024)
+ ))
+ patternIndex = (patternIndex + 1) % 4
+
+ debug("\x1b[K")
+ if csum is not None:
+ csumstr = "sha256:" + csum.hexdigest()
+ if csumstr != checksum:
+ debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
+ os.remove(dest)
+ raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
+ debug("OK\n")
+ return res
+ except Exception, e:
+ debug("FAIL %s\n" % str(e))
+ raise
+
+ def _get_url(self,server, path, headers):
+ url = "https://"; + server + path
+ debug("Fetching %s..." % url)
+
+ req = urllib2.Request(url=url)
+ if json:
+ req.add_header("Accept", "application/json")
+ for h in headers.keys():
+ req.add_header(h, headers[h])
+
+ #www Auth header starts
+ if self.www_auth_username is not None:
+ base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '')
+ req.add_header("Authorization", "Basic %s" % base64string)
+ #www Auth header finish
+
+ return urllib2.urlopen(req)
+
+ def _get_json(self,server, path, headers):
+ try:
+ res = self._get_url(server, path, headers)
+ data = json.loads(res.read())
+ debug("OK\n")
+ return (data, res)
+ except Exception, e:
+ debug("FAIL %s\n" % str(e))
+ raise
+
+def debug(msg):
+ sys.stderr.write(msg)
diff --git a/libvirt-sandbox/image/sources/Makefile.am b/libvirt-sandbox/image/sources/Makefile.am
index 48d0f33..069557d 100644
--- a/libvirt-sandbox/image/sources/Makefile.am
+++ b/libvirt-sandbox/image/sources/Makefile.am
@@ -3,6 +3,7 @@ pythonimagedir = $(pythondir)/libvirt_sandbox/image/sources
pythonimage_DATA = \
__init__.py \
Source.py \
+ DockerSource.py \
$(NULL)
EXTRA_DIST = $(pythonimage_DATA)
diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py
index f12b0eb..81f5176 100644
--- a/libvirt-sandbox/image/sources/Source.py
+++ b/libvirt-sandbox/image/sources/Source.py
@@ -31,3 +31,18 @@ class Source():
__metaclass__ = ABCMeta
def __init__(self):
pass
+
+ @abstractmethod
+ def download_template(self, templatename, templatedir,
+ registry=None, username=None, password=None):
+ """
+ :param templatename: name of the template image to download
+ :param templatedir: local directory path in which to store the template
+ :param registry: optional hostname of image registry server
+ :param username: optional username to authenticate against registry server
+ :param password: optional password to authenticate against registry server
+
+ Download a template from the registry, storing it in the local
+ filesystem
+ """
+ pass
--
2.4.3
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list