On Tue, 2015-09-08 at 17:29 +0100, Daniel P. Berrange wrote: > From: Eren Yagdiran <erenyagdiran@xxxxxxxxx> > > Refactor download function from virt-sandbox-image to use > the newly introduced Source abstract class. The docker-specific > download code is moved to a new DockerSource class. > > Signed-off-by: Daniel P. Berrange <berrange@xxxxxxxxxx> > --- > libvirt-sandbox/image/cli.py | 204 ++++--------------------- > libvirt-sandbox/image/sources/DockerSource.py | 209 ++++++++++++++++++++++++++ > libvirt-sandbox/image/sources/Makefile.am | 1 + > libvirt-sandbox/image/sources/Source.py | 15 ++ > 4 files changed, 257 insertions(+), 172 deletions(-) > create mode 100644 libvirt-sandbox/image/sources/DockerSource.py > > diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py > index de34321..7af617e 100755 > --- a/libvirt-sandbox/image/cli.py > +++ b/libvirt-sandbox/image/cli.py > @@ -69,176 +69,6 @@ def debug(msg): > def info(msg): > sys.stdout.write(msg) > > -def get_url(server, path, headers): > - url = "https://"; + server + path > - debug(" Fetching %s..." % url) > - req = urllib2.Request(url=url) > - > - if json: > - req.add_header("Accept", "application/json") > - > - for h in headers.keys(): > - req.add_header(h, headers[h]) > - > - return urllib2.urlopen(req) > - > -def get_json(server, path, headers): > - try: > - res = get_url(server, path, headers) > - data = json.loads(res.read()) > - debug("OK\n") > - return (data, res) > - except Exception, e: > - debug("FAIL %s\n" % str(e)) > - raise > - > -def save_data(server, path, headers, dest, checksum=None, datalen=None): > - try: > - res = get_url(server, path, headers) > - > - csum = None > - if checksum is not None: > - csum = hashlib.sha256() > - > - pattern = [".", "o", "O", "o"] > - patternIndex = 0 > - donelen = 0 > - > - with open(dest, "w") as f: > - while 1: > - buf = res.read(1024*64) > - if not buf: > - break > - if csum is not None: > - csum.update(buf) > - f.write(buf) > - > - if datalen is not None: > - donelen = donelen + len(buf) > - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( > - pattern[patternIndex], (donelen/1024), (datalen/1024) > - )) > - patternIndex = (patternIndex + 1) % 4 > - > - debug("\x1b[K") > - if csum is not None: > - csumstr = "sha256:" + csum.hexdigest() > - if csumstr != checksum: > - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) > - os.remove(dest) > - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) > - debug("OK\n") > - return res > - except Exception, e: > - debug("FAIL %s\n" % str(e)) > - raise > - > - > -def download_template(name, server, destdir): > - tag = "latest" > - > - offset = name.find(':') > - if offset != -1: > - tag = name[offset + 1:] > - name = name[0:offset] > - > - # First we must ask the index server about the image name. THe > - # index server will return an auth token we can use when talking > - # to the registry server. We need this token even when anonymous > - try: > - (data, res) = get_json(server, "/v1/repositories/" + name + "/images", > - {"X-Docker-Token": "true"}) > - except urllib2.HTTPError, e: > - raise ValueError(["Image '%s' does not exist" % name]) > - > - registryserver = res.info().getheader('X-Docker-Endpoints') > - token = res.info().getheader('X-Docker-Token') > - checksums = {} > - for layer in data: > - pass > - # XXX Checksums here don't appear to match the data in > - # image download later. Find out what these are sums of > - #checksums[layer["id"]] = layer["checksum"] > - > - # Now we ask the registry server for the list of tags associated > - # with the image. Tags usually reflect some kind of version of > - # the image, but they aren't officially "versions". There is > - # always a "latest" tag which is the most recent upload > - # > - # We must pass in the auth token from the index server. This > - # token can only be used once, and we're given a cookie back > - # in return to use for later RPC calls. > - (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", > - { "Authorization": "Token " + token }) > - > - cookie = res.info().getheader('Set-Cookie') > - > - if not tag in data: > - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) > - imagetagid = data[tag] > - > - # Only base images are self-contained, most images reference one > - # or more parents, in a linear stack. Here we are getting the list > - # of layers for the image with the tag we used. > - (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", > - { "Authorization": "Token " + token }) > - > - if data[0] != imagetagid: > - raise ValueError(["Expected first layer id '%s' to match image id '%s'", > - data[0], imagetagid]) > - > - try: > - createdFiles = [] > - createdDirs = [] > - > - for layerid in data: > - templatedir = destdir + "/" + layerid > - if not os.path.exists(templatedir): > - os.mkdir(templatedir) > - createdDirs.append(templatedir) > - > - jsonfile = templatedir + "/template.json" > - datafile = templatedir + "/template.tar.gz" > - > - if not os.path.exists(jsonfile) or not os.path.exists(datafile): > - # The '/json' URL gives us some metadata about the layer > - res = save_data(registryserver, "/v1/images/" + layerid + "/json", > - { "Authorization": "Token " + token }, jsonfile) > - createdFiles.append(jsonfile) > - layersize = int(res.info().getheader("Content-Length")) > - > - datacsum = None > - if layerid in checksums: > - datacsum = checksums[layerid] > - > - # and the '/layer' URL is the actual payload, provided > - # as a tar.gz archive > - save_data(registryserver, "/v1/images/" + layerid + "/layer", > - { "Authorization": "Token " + token }, datafile, datacsum, layersize) > - createdFiles.append(datafile) > - > - # Strangely the 'json' data for a layer doesn't include > - # its actual name, so we save that in a json file of our own > - index = { > - "name": name, > - } > - > - indexfile = destdir + "/" + imagetagid + "/index.json" > - with open(indexfile, "w") as f: > - f.write(json.dumps(index)) > - except Exception, e: > - for f in createdFiles: > - try: > - os.remove(f) > - except: > - pass > - for d in createdDirs: > - try: > - os.rmdir(d) > - except: > - pass > - > - > def delete_template(name, destdir): > imageusage = {} > imageparent = {} > @@ -342,8 +172,16 @@ def create_template(name, imagepath, format, destdir): > parentImage = templateImage > > def download(args): > - info("Downloading %s from %s to %s\n" % (args.template, default_index_server, default_template_dir)) > - download_template(args.template, default_index_server, default_template_dir) > + try: > + dynamic_source_loader(args.source).download_template(templatename=args.template, > + templatedir=args.template_dir, > + registry=args.registry, > + username=args.username, > + password=args.password) > + except IOError,e: > + print "Source %s cannot be found in given path" %args.source > + except Exception,e: > + print "Download Error %s" % str(e) > > def delete(args): > info("Deleting %s from %s\n" % (args.template, default_template_dir)) > @@ -357,10 +195,32 @@ def requires_template(parser): > parser.add_argument("template", > help=_("name of the template")) > > +def requires_source(parser): > + parser.add_argument("-s","--source", > + default="docker", > + help=_("name of the template")) > + > +def requires_auth_conn(parser): > + parser.add_argument("-r","--registry", > + help=_("Url of the custom registry")) This wording really sounds docker-specific. The registry word only fits docker terminology, would surely not apply to virt-builder or appc case. Something like "images storage" would may be more generic. The problem is that "repository" has a special meaning in the docker terminology. > + parser.add_argument("-u","--username", > + help=_("Username for the custom registry")) > + parser.add_argument("-p","--password", > + help=_("Password for the custom registry")) > + > +def requires_template_dir(parser): > + global default_template_dir > + parser.add_argument("-t","--template-dir", > + default=default_template_dir, > + help=_("Template directory for saving templates")) > + > def gen_download_args(subparser): > parser = subparser.add_parser("download", > help=_("Download template data")) > requires_template(parser) > + requires_source(parser) > + requires_auth_conn(parser) > + requires_template_dir(parser) > parser.set_defaults(func=download) > > def gen_delete_args(subparser): > diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py > new file mode 100644 > index 0000000..37b40dc > --- /dev/null > +++ b/libvirt-sandbox/image/sources/DockerSource.py > @@ -0,0 +1,209 @@ > +#!/usr/bin/python > +# -*- coding: utf-8 -*- > +# > +# Copyright (C) 2015 Universitat Politècnica de Catalunya. > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > +# > +# Author: Eren Yagdiran <erenyagdiran@xxxxxxxxx> > +# > + > +from Source import Source > +import urllib2 > +import sys > +import json > +import traceback > +import os > +import subprocess > +import shutil > + > +class DockerSource(Source): > + > + www_auth_username = None > + www_auth_password = None > + > + def __init__(self): > + self.default_index_server = "index.docker.io" > + > + def _check_cert_validate(self): > + major = sys.version_info.major > + SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" > + SSL_WARNING +="\nSee https://bugs.python.org/issue22417\n"; > + py2_7_9_hexversion = 34015728 > + py3_4_3_hexversion = 50594800 > + if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): > + sys.stderr.write(SSL_WARNING) > + > + def download_template(self, templatename, templatedir, > + registry=None, username=None, password=None): > + if registry is None: > + registry = self.default_index_server > + > + if username is not None: > + self.www_auth_username = username > + self.www_auth_password = password > + > + self._check_cert_validate() > + tag = "latest" > + offset = templatename.find(':') > + if offset != -1: > + tag = templatename[offset + 1:] > + templatename = templatename[0:offset] > + try: > + (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", > + {"X-Docker-Token": "true"}) > + except urllib2.HTTPError, e: > + raise ValueError(["Image '%s' does not exist" % templatename]) > + > + registryendpoint = res.info().getheader('X-Docker-Endpoints') > + token = res.info().getheader('X-Docker-Token') > + checksums = {} > + for layer in data: > + pass > + (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", > + { "Authorization": "Token " + token }) > + > + cookie = res.info().getheader('Set-Cookie') > + > + if not tag in data: > + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) > + imagetagid = data[tag] > + > + (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", > + { "Authorization": "Token "+token }) > + > + if data[0] != imagetagid: > + raise ValueError(["Expected first layer id '%s' to match image id '%s'", > + data[0], imagetagid]) > + > + try: > + createdFiles = [] > + createdDirs = [] > + > + for layerid in data: > + layerdir = templatedir + "/" + layerid > + if not os.path.exists(layerdir): > + os.makedirs(layerdir) > + createdDirs.append(layerdir) > + > + jsonfile = layerdir + "/template.json" > + datafile = layerdir + "/template.tar.gz" > + > + if not os.path.exists(jsonfile) or not os.path.exists(datafile): > + res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", > + { "Authorization": "Token " + token }, jsonfile) > + createdFiles.append(jsonfile) > + > + layersize = int(res.info().getheader("Content-Length")) > + > + datacsum = None > + if layerid in checksums: > + datacsum = checksums[layerid] > + > + self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", > + { "Authorization": "Token "+token }, datafile, datacsum, layersize) > + createdFiles.append(datafile) > + > + index = { > + "name": templatename, > + } > + > + indexfile = templatedir + "/" + imagetagid + "/index.json" > + print("Index file " + indexfile) > + with open(indexfile, "w") as f: > + f.write(json.dumps(index)) > + except Exception as e: > + traceback.print_exc() > + for f in createdFiles: > + try: > + os.remove(f) > + except: > + pass > + for d in createdDirs: > + try: > + shutil.rmtree(d) > + except: > + pass > + def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): > + try: > + res = self._get_url(server, path, headers) > + > + csum = None > + if checksum is not None: > + csum = hashlib.sha256() > + > + pattern = [".", "o", "O", "o"] > + patternIndex = 0 > + donelen = 0 > + > + with open(dest, "w") as f: > + while 1: > + buf = res.read(1024*64) > + if not buf: > + break > + if csum is not None: > + csum.update(buf) > + f.write(buf) > + > + if datalen is not None: > + donelen = donelen + len(buf) > + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( > + pattern[patternIndex], (donelen/1024), (datalen/1024) > + )) > + patternIndex = (patternIndex + 1) % 4 > + > + debug("\x1b[K") > + if csum is not None: > + csumstr = "sha256:" + csum.hexdigest() > + if csumstr != checksum: > + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) > + os.remove(dest) > + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) > + debug("OK\n") > + return res > + except Exception, e: > + debug("FAIL %s\n" % str(e)) > + raise > + > + def _get_url(self,server, path, headers): > + url = "https://"; + server + path > + debug("Fetching %s..." % url) > + > + req = urllib2.Request(url=url) > + if json: > + req.add_header("Accept", "application/json") > + for h in headers.keys(): > + req.add_header(h, headers[h]) > + > + #www Auth header starts > + if self.www_auth_username is not None: > + base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') > + req.add_header("Authorization", "Basic %s" % base64string) > + #www Auth header finish > + > + return urllib2.urlopen(req) > + > + def _get_json(self,server, path, headers): > + try: > + res = self._get_url(server, path, headers) > + data = json.loads(res.read()) > + debug("OK\n") > + return (data, res) > + except Exception, e: > + debug("FAIL %s\n" % str(e)) > + raise > + > +def debug(msg): > + sys.stderr.write(msg) > diff --git a/libvirt-sandbox/image/sources/Makefile.am b/libvirt-sandbox/image/sources/Makefile.am > index 48d0f33..069557d 100644 > --- a/libvirt-sandbox/image/sources/Makefile.am > +++ b/libvirt-sandbox/image/sources/Makefile.am > @@ -3,6 +3,7 @@ pythonimagedir = $(pythondir)/libvirt_sandbox/image/sources > pythonimage_DATA = \ > __init__.py \ > Source.py \ > + DockerSource.py \ > $(NULL) > > EXTRA_DIST = $(pythonimage_DATA) > diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py > index f12b0eb..81f5176 100644 > --- a/libvirt-sandbox/image/sources/Source.py > +++ b/libvirt-sandbox/image/sources/Source.py > @@ -31,3 +31,18 @@ class Source(): > __metaclass__ = ABCMeta > def __init__(self): > pass > + > + @abstractmethod > + def download_template(self, templatename, templatedir, > + registry=None, username=None, password=None): > + """ > + :param templatename: name of the template image to download > + :param templatedir: local directory path in which to store the template > + :param registry: optional hostname of image registry server > + :param username: optional username to authenticate against registry server > + :param password: optional password to authenticate against registry server > + > + Download a template from the registry, storing it in the local > + filesystem > + """ > + pass ACK, but may need some thinking on the "Registry" word. -- Cedric -- libvir-list mailing list libvir-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/libvir-list