On Mon, 2015-09-21 at 15:45 +0100, Daniel P. Berrange wrote: > Currently the CLI syntax is somewhat docker specific requiring > inclusion of --registry arg to identify the docker download > server. Other app containers have a notion of download server, > but don't separate it from the template name. > > This patch removes that docker-ism by changing to use a URI > for identifying the template image. So instead of > > virt-sandbox-image download \ > --source docker --registry index.docker.io > --username dan --password 123456 ubuntu:15.04 > > You can use > > virt-sandbox-image download docker://dan:123456@xxxxxxxxxxxxxxx/ubuntu?tag=15.04 > > The only mandatory part is the source prefix and image name, so > that can shorten to just > > virt-sandbox-image download docker:///ubuntu > > to pull down the latest ubuntu image, from the default registry > using no authentication. > --- > > Changed in v2: > > - Rebase against master, instead of (unpushed) docker volume patch > > libvirt-sandbox/image/cli.py | 71 +++++-------- > libvirt-sandbox/image/sources/DockerSource.py | 142 ++++++++++++++------------ > libvirt-sandbox/image/sources/Source.py | 29 +++--- > libvirt-sandbox/image/template.py | 110 ++++++++++++++++++++ > 4 files changed, 228 insertions(+), 124 deletions(-) > create mode 100644 libvirt-sandbox/image/template.py > > diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py > index 1718cc5..4d02289 100755 > --- a/libvirt-sandbox/image/cli.py > +++ b/libvirt-sandbox/image/cli.py > @@ -3,7 +3,7 @@ > # Authors: Daniel P. Berrange <berrange@xxxxxxxxxx> > # Eren Yagdiran <erenyagdiran@xxxxxxxxx> > # > -# Copyright (C) 2013 Red Hat, Inc. > +# Copyright (C) 2013-2015 Red Hat, Inc. > # Copyright (C) 2015 Universitat Politècnica de Catalunya. > # > # This program is free software; you can redistribute it and/or modify > @@ -34,6 +34,8 @@ import subprocess > import random > import string > > +from libvirt_sandbox.image import template > + > if os.geteuid() == 0: > default_template_dir = "/var/lib/libvirt/templates" > default_image_dir = "/var/lib/libvirt/images" > @@ -44,15 +46,6 @@ else: > debug = False > verbose = False > > -import importlib > -def dynamic_source_loader(name): > - name = name[0].upper() + name[1:] > - modname = "libvirt_sandbox.image.sources." + name + "Source" > - mod = importlib.import_module(modname) > - classname = name + "Source" > - classimpl = getattr(mod, classname) > - return classimpl() > - > gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") > gettext.textdomain("libvirt-sandbox") > try: > @@ -73,11 +66,10 @@ def info(msg): > > def download(args): > try: > - dynamic_source_loader(args.source).download_template(templatename=args.template, > - templatedir=args.template_dir, > - registry=args.registry, > - username=args.username, > - password=args.password) > + tmpl = template.Template.from_uri(args.template) > + source = tmpl.get_source_impl() > + source.download_template(template=tmpl, > + templatedir=args.template_dir) > except IOError,e: > print "Source %s cannot be found in given path" %args.source > except Exception,e: > @@ -85,17 +77,21 @@ def download(args): > > def delete(args): > try: > - dynamic_source_loader(args.source).delete_template(templatename=args.template, > - templatedir=args.template_dir) > + tmpl = template.Template.from_uri(args.template) > + source = tmpl.get_source_impl() > + source.delete_template(template=tmpl, > + templatedir=args.template_dir) > except Exception,e: > print "Delete Error %s", str(e) > > def create(args): > try: > - dynamic_source_loader(args.source).create_template(templatename=args.template, > - templatedir=args.template_dir, > - connect=args.connect, > - format=args.format) > + tmpl = template.Template.from_uri(args.template) > + source = tmpl.get_source_impl() > + source.create_template(template=tmpl, > + templatedir=args.template_dir, > + connect=args.connect, > + format=args.format) > except Exception,e: > print "Create Error %s" % str(e) > > @@ -103,19 +99,22 @@ def run(args): > try: > if args.connect is not None: > check_connect(args.connect) > - source = dynamic_source_loader(args.source) > + > + tmpl = template.Template.from_uri(args.template) > + source = tmpl.get_source_impl() > + > name = args.name > if name is None: > randomid = ''.join(random.choice(string.lowercase) for i in range(10)) > - name = args.template + ":" + randomid > + name = tmpl.path[1:] + ":" + randomid > > - diskfile = source.get_disk(templatename=args.template, > + diskfile = source.get_disk(template=tmpl, > templatedir=args.template_dir, > imagedir=args.image_dir, > sandboxname=name) > > format = "qcow2" > - commandToRun = source.get_command(args.template, args.template_dir, args.args) > + commandToRun = source.get_command(tmpl, args.template_dir, args.args) > if len(commandToRun) == 0: > commandToRun = ["/bin/sh"] > cmd = ['virt-sandbox', '--name', name] > @@ -129,7 +128,7 @@ def run(args): > params.append('-N') > params.append(networkArgs) > > - allEnvs = source.get_env(args.template, args.template_dir) > + allEnvs = source.get_env(tmpl, args.template_dir) > envArgs = args.env > if envArgs is not None: > allEnvs = allEnvs + envArgs > @@ -151,7 +150,7 @@ def run(args): > > def requires_template(parser): > parser.add_argument("template", > - help=_("name of the template")) > + help=_("URI of the template")) > > def requires_name(parser): > parser.add_argument("-n","--name", > @@ -163,23 +162,10 @@ def check_connect(connectstr): > raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) > return True > > -def requires_source(parser): > - parser.add_argument("-s","--source", > - default="docker", > - help=_("name of the template")) > - > def requires_connect(parser): > parser.add_argument("-c","--connect", > help=_("Connect string for libvirt")) > > -def requires_auth_conn(parser): > - parser.add_argument("-r","--registry", > - help=_("Url of the custom registry")) > - parser.add_argument("-u","--username", > - help=_("Username for the custom registry")) > - parser.add_argument("-p","--password", > - help=_("Password for the custom registry")) > - > def requires_template_dir(parser): > global default_template_dir > parser.add_argument("-t","--template-dir", > @@ -196,8 +182,6 @@ def gen_download_args(subparser): > parser = subparser.add_parser("download", > help=_("Download template data")) > requires_template(parser) > - requires_source(parser) > - requires_auth_conn(parser) > requires_template_dir(parser) > parser.set_defaults(func=download) > > @@ -205,7 +189,6 @@ def gen_delete_args(subparser): > parser = subparser.add_parser("delete", > help=_("Delete template data")) > requires_template(parser) > - requires_source(parser) > requires_template_dir(parser) > parser.set_defaults(func=delete) > > @@ -213,7 +196,6 @@ def gen_create_args(subparser): > parser = subparser.add_parser("create", > help=_("Create image from template data")) > requires_template(parser) > - requires_source(parser) > requires_connect(parser) > requires_template_dir(parser) > parser.add_argument("-f","--format", > @@ -226,7 +208,6 @@ def gen_run_args(subparser): > help=_("Run an already built image")) > requires_name(parser) > requires_template(parser) > - requires_source(parser) > requires_connect(parser) > requires_template_dir(parser) > requires_image_dir(parser) > diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py > index c374a0c..10f8537 100644 > --- a/libvirt-sandbox/image/sources/DockerSource.py > +++ b/libvirt-sandbox/image/sources/DockerSource.py > @@ -2,6 +2,7 @@ > # -*- coding: utf-8 -*- > # > # Copyright (C) 2015 Universitat Politècnica de Catalunya. > +# Copyright (C) 2015 Red Hat, Inc > # > # This library is free software; you can redistribute it and/or > # modify it under the terms of the GNU Lesser General Public > @@ -28,6 +29,8 @@ import traceback > import os > import subprocess > import shutil > +import urlparse > + > > class DockerConfParser(): > > @@ -47,12 +50,6 @@ class DockerConfParser(): > > class DockerSource(Source): > > - www_auth_username = None > - www_auth_password = None > - > - def __init__(self): > - self.default_index_server = "index.docker.io" > - > def _check_cert_validate(self): > major = sys.version_info.major > SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" > @@ -62,43 +59,38 @@ class DockerSource(Source): > if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): > sys.stderr.write(SSL_WARNING) > > - def download_template(self, templatename, templatedir, > - registry=None, username=None, password=None): > - if registry is None: > - registry = self.default_index_server > - > - if username is not None: > - self.www_auth_username = username > - self.www_auth_password = password > - > + def download_template(self, template, templatedir): > self._check_cert_validate() > - tag = "latest" > - offset = templatename.find(':') > - if offset != -1: > - tag = templatename[offset + 1:] > - templatename = templatename[0:offset] > + > try: > - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", > - {"X-Docker-Token": "true"}) > + (data, res) = self._get_json(template, > + None, > + "/v1/repositories" + template.path + "/images", > + {"X-Docker-Token": "true"}) > except urllib2.HTTPError, e: > - raise ValueError(["Image '%s' does not exist" % templatename]) > + raise ValueError(["Image '%s' does not exist" % template]) > > registryendpoint = res.info().getheader('X-Docker-Endpoints') > token = res.info().getheader('X-Docker-Token') > checksums = {} > for layer in data: > pass > - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", > - { "Authorization": "Token " + token }) > + (data, res) = self._get_json(template, > + registryendpoint, > + "/v1/repositories" + template.path + "/tags", > + { "Authorization": "Token " + token }) > > cookie = res.info().getheader('Set-Cookie') > > + tag = template.params.get("tag", "latest") > if not tag in data: > - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) > + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) > imagetagid = data[tag] > > - (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", > - { "Authorization": "Token "+token }) > + (data, res) = self._get_json(template, > + registryendpoint, > + "/v1/images/" + imagetagid + "/ancestry", > + { "Authorization": "Token "+ token }) > > if data[0] != imagetagid: > raise ValueError(["Expected first layer id '%s' to match image id '%s'", > @@ -118,8 +110,11 @@ class DockerSource(Source): > datafile = layerdir + "/template.tar.gz" > > if not os.path.exists(jsonfile) or not os.path.exists(datafile): > - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", > - { "Authorization": "Token " + token }, jsonfile) > + res = self._save_data(template, > + registryendpoint, > + "/v1/images/" + layerid + "/json", > + { "Authorization": "Token " + token }, > + jsonfile) > createdFiles.append(jsonfile) > > layersize = int(res.info().getheader("Content-Length")) > @@ -128,12 +123,15 @@ class DockerSource(Source): > if layerid in checksums: > datacsum = checksums[layerid] > > - self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", > - { "Authorization": "Token "+token }, datafile, datacsum, layersize) > + self._save_data(template, > + registryendpoint, > + "/v1/images/" + layerid + "/layer", > + { "Authorization": "Token "+token }, > + datafile, datacsum, layersize) > createdFiles.append(datafile) > > index = { > - "name": templatename, > + "name": template.path, > } > > indexfile = templatedir + "/" + imagetagid + "/index.json" > @@ -152,9 +150,11 @@ class DockerSource(Source): > shutil.rmtree(d) > except: > pass > - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): > + > + def _save_data(self, template, server, path, headers, > + dest, checksum=None, datalen=None): > try: > - res = self._get_url(server, path, headers) > + res = self._get_url(template, server, path, headers) > > csum = None > if checksum is not None: > @@ -193,8 +193,22 @@ class DockerSource(Source): > debug("FAIL %s\n" % str(e)) > raise > > - def _get_url(self,server, path, headers): > - url = "https://" + server + path > + def _get_url(self, template, server, path, headers): > + if template.protocol is None: > + protocol = "https" > + else: > + protocol = template.protocol > + > + if server is None: > + if template.hostname is None: > + server = "index.docker.io" > + else: > + if template.port is not None: > + server = template.hostname + ":" + template.port This doesn't fly, port is an int, we need to have it in this form: server = "%s:%d" % (template.hostname, template.port) > + else: > + server = template.hostname > + > + url = urlparse.urlunparse((protocol, server, path, None, None, None)) > debug("Fetching %s..." % url) > > req = urllib2.Request(url=url) > @@ -204,16 +218,18 @@ class DockerSource(Source): > req.add_header(h, headers[h]) > > #www Auth header starts > - if self.www_auth_username is not None: > - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') > + if template.username and template.password: > + base64string = base64.encodestring( > + '%s:%s' % (template.username, > + template.password)).replace('\n', '') > req.add_header("Authorization", "Basic %s" % base64string) > #www Auth header finish > > return urllib2.urlopen(req) > > - def _get_json(self,server, path, headers): > + def _get_json(self, template, server, path, headers): > try: > - res = self._get_url(server, path, headers) > + res = self._get_url(template, server, path, headers) > data = json.loads(res.read()) > debug("OK\n") > return (data, res) > @@ -221,11 +237,11 @@ class DockerSource(Source): > debug("FAIL %s\n" % str(e)) > raise > > - def create_template(self, templatename, templatedir, connect=None, format=None): > + def create_template(self, template, templatedir, connect=None, format=None): > if format is None: > format = self.default_disk_format > self._check_disk_format(format) > - imagelist = self._get_image_list(templatename,templatedir) > + imagelist = self._get_image_list(template, templatedir) > imagelist.reverse() > > parentImage = None > @@ -252,7 +268,7 @@ class DockerSource(Source): > if not format in supportedFormats: > raise ValueError(["Unsupported image format %s" % format]) > > - def _get_image_list(self,templatename,destdir): > + def _get_image_list(self, template, destdir): > imageparent = {} > imagenames = {} > imagedirs = os.listdir(destdir) > @@ -265,13 +281,13 @@ class DockerSource(Source): > jsonfile = destdir + "/" + imagetagid + "/template.json" > if os.path.exists(jsonfile): > with open(jsonfile,"r") as f: > - template = json.load(f) > - parent = template.get("parent",None) > + data = json.load(f) > + parent = data.get("parent",None) > if parent: > imageparent[imagetagid] = parent > - if not templatename in imagenames: > - raise ValueError(["Image %s does not exist locally" %templatename]) > - imagetagid = imagenames[templatename] > + if not template.path in imagenames: > + raise ValueError(["Image %s does not exist locally" % template.path]) > + imagetagid = imagenames[template.path] > imagelist = [] > while imagetagid != None: > imagelist.append(imagetagid) > @@ -310,7 +326,7 @@ class DockerSource(Source): > cmd = cmd + params > subprocess.call(cmd) > > - def delete_template(self, templatename, templatedir): > + def delete_template(self, template, templatedir): > imageusage = {} > imageparent = {} > imagenames = {} > @@ -324,9 +340,9 @@ class DockerSource(Source): > jsonfile = templatedir + "/" + imagetagid + "/template.json" > if os.path.exists(jsonfile): > with open(jsonfile,"r") as f: > - template = json.load(f) > + data = json.load(f) > > - parent = template.get("parent",None) > + parent = data.get("parent",None) > if parent: > if parent not in imageusage: > imageusage[parent] = [] > @@ -334,10 +350,10 @@ class DockerSource(Source): > imageparent[imagetagid] = parent > > > - if not templatename in imagenames: > - raise ValueError(["Image %s does not exist locally" %templatename]) > + if not template.path in imagenames: > + raise ValueError(["Image %s does not exist locally" % template.path]) > > - imagetagid = imagenames[templatename] > + imagetagid = imagenames[template.path] > while imagetagid != None: > debug("Remove %s\n" % imagetagid) > parent = imageparent.get(imagetagid,None) > @@ -360,15 +376,15 @@ class DockerSource(Source): > parent = None > imagetagid = parent > > - def _get_template_data(self, templatename, templatedir): > - imageList = self._get_image_list(templatename, templatedir) > + def _get_template_data(self, template, templatedir): > + imageList = self._get_image_list(template, templatedir) > toplayer = imageList[0] > diskfile = templatedir + "/" + toplayer + "/template.qcow2" > configfile = templatedir + "/" + toplayer + "/template.json" > return configfile, diskfile > > - def get_disk(self,templatename, templatedir, imagedir, sandboxname): > - configfile, diskfile = self._get_template_data(templatename, templatedir) > + def get_disk(self, template, templatedir, imagedir, sandboxname): > + configfile, diskfile = self._get_template_data(template, templatedir) > tempfile = imagedir + "/" + sandboxname + ".qcow2" > if not os.path.exists(imagedir): > os.makedirs(imagedir) > @@ -379,8 +395,8 @@ class DockerSource(Source): > subprocess.call(cmd) > return tempfile > > - def get_command(self, templatename, templatedir, userargs): > - configfile, diskfile = self._get_template_data(templatename, templatedir) > + def get_command(self, template, templatedir, userargs): > + configfile, diskfile = self._get_template_data(template, templatedir) > configParser = DockerConfParser(configfile) > cmd = configParser.getCommand() > entrypoint = configParser.getEntrypoint() > @@ -393,8 +409,8 @@ class DockerSource(Source): > else: > return entrypoint + cmd > > - def get_env(self, templatename, templatedir): > - configfile, diskfile = self._get_template_data(templatename, templatedir) > + def get_env(self, template, templatedir): > + configfile, diskfile = self._get_template_data(template, templatedir) > configParser = DockerConfParser(configfile) > return configParser.getEnvs() > > diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py > index 8a21f90..597a7fb 100644 > --- a/libvirt-sandbox/image/sources/Source.py > +++ b/libvirt-sandbox/image/sources/Source.py > @@ -2,6 +2,7 @@ > # -*- coding: utf-8 -*- > # > # Copyright (C) 2015 Universitat Politècnica de Catalunya. > +# Copyright (C) 2015 Red Hat, Inc > # > # This library is free software; you can redistribute it and/or > # modify it under the terms of the GNU Lesser General Public > @@ -33,14 +34,10 @@ class Source(): > pass > > @abstractmethod > - def download_template(self, templatename, templatedir, > - registry=None, username=None, password=None): > + def download_template(self, template, templatedir): > """ > - :param templatename: name of the template image to download > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path in which to store the template > - :param registry: optional hostname of image registry server > - :param username: optional username to authenticate against registry server > - :param password: optional password to authenticate against registry server > > Download a template from the registry, storing it in the local > filesystem > @@ -48,10 +45,10 @@ class Source(): > pass > > @abstractmethod > - def create_template(self, templatename, templatedir, > + def create_template(self, template, templatedir, > connect=None, format=None): > """ > - :param templatename: name of the template image to create > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path in which to store the template > :param connect: libvirt connection URI > :param format: disk image format > @@ -63,9 +60,9 @@ class Source(): > pass > > @abstractmethod > - def delete_template(self, templatename, templatedir): > + def delete_template(self, template, templatedir): > """ > - :param templatename: name of the template image to delete > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path from which to delete template > > Delete all local files associated with the template > @@ -73,9 +70,9 @@ class Source(): > pass > > @abstractmethod > - def get_command(self, templatename, templatedir, userargs): > + def get_command(self, template, templatedir, userargs): > """ > - :param templatename: name of the template image to query > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path in which templates are stored > :param userargs: user specified arguments to run > > @@ -85,9 +82,9 @@ class Source(): > pass > > @abstractmethod > - def get_disk(self,templatename, templatedir, imagedir, sandboxname): > + def get_disk(self, template, templatedir, imagedir, sandboxname): > """ > - :param templatename: name of the template image to download > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path in which to find template > :param imagedir: local directory in which to storage disk image > > @@ -97,9 +94,9 @@ class Source(): > pass > > @abstractmethod > - def get_env(self,templatename, templatedir): > + def get_env(self, template, templatedir): > """ > - :param templatename: name of the template image to download > + :param template: libvirt_sandbox.template.Template object > :param templatedir: local directory path in which to find template > > Get the dict of environment variables to set > diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py > new file mode 100644 > index 0000000..0ad767b > --- /dev/null > +++ b/libvirt-sandbox/image/template.py > @@ -0,0 +1,110 @@ > +# > +# -*- coding: utf-8 -*- > +# Authors: Daniel P. Berrange <berrange@xxxxxxxxxx> > +# > +# Copyright (C) 2015 Red Hat, Inc. > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; either version 2 of the License, or > +# (at your option) any later version. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write to the Free Software > +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. > +# > + > +import urlparse > +import importlib > + > +class Template(object): > + > + def __init__(self, > + source, protocol, > + hostname, port, > + username, password, > + path, params): > + """ > + :param source: template source name > + :param protocol: network transport protocol or None > + :param hostname: registry hostname or None > + :param port: registry port or None > + :param username: username or None > + :param password: password or None > + :param path: template path identifier > + :param params: template parameters > + > + docker:///ubuntu > + > + docker+https://index.docker.io/ubuntu?tag=latest > + """ > + > + self.source = source > + self.protocol = protocol > + self.hostname = hostname > + self.port = port > + self.username = username > + self.password = password > + self.path = path > + self.params = params > + if self.params is None: > + self.params = {} > + > + def get_source_impl(self): > + mod = importlib.import_module( > + "libvirt_sandbox.image.sources." + > + self.source.capitalize() + "Source") > + classname = self.source.capitalize() + "Source" > + classimpl = getattr(mod, classname) > + return classimpl() > + > + def __repr__(self): > + if self.protocol is not None: > + scheme = self.source + "+" + self.protocol > + else: > + scheme = self.source > + if self.hostname: > + if self.port: > + netloc = self.hostname + ":" + self.port This doesn't work, python requires to put it this way: netloc = "%s:%d" % (self.hostname, self.port) -- Cedric > + else: > + netloc = self.hostname > + > + if self.username: > + if self.password: > + auth = self.username + ":" + self.password > + else: > + auth = self.username > + netloc = auth + "@" + netloc > + else: > + netloc = None > + > + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) > + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) > + > + @classmethod > + def from_uri(klass, uri): > + o = urlparse.urlparse(uri) > + > + idx = o.scheme.find("+") > + if idx == -1: > + source = o.scheme > + protocol = None > + else: > + source = o.scheme[0:idx] > + protocol = o.schema[idx + 1:] s/schema/scheme/ > + > + query = {} > + if o.query is not None and o.query != "": > + for param in o.query.split("&"): > + (key, val) = param.split("=") > + query[key] = val > + return klass(source, protocol, > + o.hostname, o.port, > + o.username, o.password, > + o.path, query) > + -- libvir-list mailing list libvir-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/libvir-list