[PATCH sandbox 20/24] docker: introduce a DockerRegistry class

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Introduce a class to handle HTTP requests with a docker registry
server, and associated auth credentials.

Signed-off-by: Daniel P. Berrange <berrange@xxxxxxxxxx>
---
 libvirt-sandbox/image/sources/docker.py | 264 ++++++++++++++++----------------
 1 file changed, 136 insertions(+), 128 deletions(-)

diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py
index a54f563..b2706b1 100644
--- a/libvirt-sandbox/image/sources/docker.py
+++ b/libvirt-sandbox/image/sources/docker.py
@@ -31,6 +31,7 @@ import shutil
 import urlparse
 import hashlib
 from abc import ABCMeta, abstractmethod
+import copy
 
 from . import base
 
@@ -152,14 +153,132 @@ class DockerAuthToken(DockerAuth):
         return False
 
 
-class DockerSource(base.Source):
+class DockerRegistry():
 
-    def __init__(self):
+    def __init__(self, uri_base):
+
+        self.uri_base = list(urlparse.urlparse(uri_base))
         self.auth_handler = DockerAuthNop()
 
     def set_auth_handler(self, auth_handler):
         self.auth_handler = auth_handler
 
+    def set_server(self, server):
+        self.uri_base[1] = server
+
+    @classmethod
+    def from_template(cls, template):
+        protocol = template.protocol
+        hostname = template.hostname
+        port = template.port
+
+        if protocol is None:
+            protocol = "https"
+        if hostname is None:
+            hostname = "index.docker.io"
+
+        if port is None:
+            server = hostname
+        else:
+            server = "%s:%s" % (hostname, port)
+
+        url = urlparse.urlunparse((protocol, server, "", None, None, None))
+
+        return cls(url)
+
+    def get_url(self, path, headers=None):
+        url_bits = copy.copy(self.uri_base)
+        url_bits[2] = path
+        url = urlparse.urlunparse(url_bits)
+        debug("Fetching %s..." % url)
+
+        req = urllib2.Request(url=url)
+
+        if headers is not None:
+            for h in headers.keys():
+                req.add_header(h, headers[h])
+
+        self.auth_handler.prepare_req(req)
+
+        try:
+            res = urllib2.urlopen(req)
+            self.auth_handler.process_res(res)
+            return res
+        except urllib2.HTTPError as e:
+            if e.code == 401:
+                retry = self.auth_handler.process_err(e)
+                if retry:
+                    debug("Re-Fetching %s..." % url)
+                    self.auth_handler.prepare_req(req)
+                    res = urllib2.urlopen(req)
+                    self.auth_handler.process_res(res)
+                    return res
+                else:
+                    debug("Not re-fetching")
+                    raise
+            else:
+                raise
+
+    def save_data(self, path, dest, checksum=None):
+        try:
+            res = self.get_url(path)
+
+            datalen = res.info().getheader("Content-Length")
+            if datalen is not None:
+                datalen = int(datalen)
+
+            csum = None
+            if checksum is not None:
+                csum = hashlib.sha256()
+
+            pattern = [".", "o", "O", "o"]
+            patternIndex = 0
+            donelen = 0
+
+            with open(dest, "w") as f:
+                while 1:
+                    buf = res.read(1024*64)
+                    if not buf:
+                        break
+                    if csum is not None:
+                        csum.update(buf)
+                    f.write(buf)
+
+                    if datalen is not None:
+                        donelen = donelen + len(buf)
+                        debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
+                            pattern[patternIndex], (donelen/1024), (datalen/1024)
+                        ))
+                        patternIndex = (patternIndex + 1) % 4
+
+            debug("\x1b[K")
+            if csum is not None:
+                csumstr = "sha256:" + csum.hexdigest()
+                if csumstr != checksum:
+                    debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
+                    os.remove(dest)
+                    raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
+            debug("OK\n")
+            return res
+        except Exception, e:
+            debug("FAIL %s\n" % str(e))
+            raise
+
+    def get_json(self, path):
+        try:
+            headers = {}
+            headers["Accept"] = "application/json"
+            res = self.get_url(path, headers)
+            data = json.loads(res.read())
+            debug("OK\n")
+            return (data, res)
+        except Exception, e:
+            debug("FAIL %s\n" % str(e))
+            raise
+
+
+class DockerSource(base.Source):
+
     def _check_cert_validate(self):
         major = sys.version_info.major
         SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
@@ -189,38 +308,33 @@ class DockerSource(base.Source):
     def download_template(self, image, template, templatedir):
         self._check_cert_validate()
 
+        registry = DockerRegistry.from_template(template)
         basicauth = DockerAuthBasic(template.username, template.password)
-        self.set_auth_handler(basicauth)
+        registry.set_auth_handler(basicauth)
         try:
-            (data, res) = self._get_json(template,
-                                         None,
-                                         "/v1/repositories/%s/%s/images" % (
-                                             image.repo, image.name,
-                                         ))
+            (data, res) = registry.get_json("/v1/repositories/%s/%s/images" % (
+                                                image.repo, image.name,
+                                            ))
         except urllib2.HTTPError, e:
             raise ValueError(["Image '%s' does not exist" % template])
 
         registryendpoint = res.info().getheader('X-Docker-Endpoints')
 
         if basicauth.token is not None:
-            self.set_auth_handler(DockerAuthToken(basicauth.token))
+            registry.set_auth_handler(DockerAuthToken(basicauth.token))
         else:
-            self.set_auth_handler(DockerAuthNop())
+            registry.set_auth_handler(DockerAuthNop())
 
-        (data, res) = self._get_json(template,
-                                     registryendpoint,
-                                     "/v1/repositories/%s/%s/tags" %(
-                                         image.repo, image.name
-                                     ))
+        (data, res) = registry.get_json("/v1/repositories/%s/%s/tags" %(
+                                            image.repo, image.name
+                                        ))
 
         if image.tag not in data:
             raise ValueError(["Tag '%s' does not exist for image '%s'" %
                               (image.tag, template)])
         imagetagid = data[image.tag]
 
-        (data, res) = self._get_json(template,
-                                     registryendpoint,
-                                     "/v1/images/" + imagetagid + "/ancestry")
+        (data, res) = registry.get_json("/v1/images/" + imagetagid + "/ancestry")
 
         if data[0] != imagetagid:
             raise ValueError(["Expected first layer id '%s' to match image id '%s'",
@@ -240,16 +354,12 @@ class DockerSource(base.Source):
                 datafile = layerdir + "/template.tar.gz"
 
                 if not os.path.exists(jsonfile) or not os.path.exists(datafile):
-                    res = self._save_data(template,
-                                          registryendpoint,
-                                          "/v1/images/" + layerid + "/json",
-                                          jsonfile)
+                    res = registry.save_data("/v1/images/" + layerid + "/json",
+                                             jsonfile)
                     createdFiles.append(jsonfile)
 
-                    self._save_data(template,
-                                    registryendpoint,
-                                    "/v1/images/" + layerid + "/layer",
-                                    datafile)
+                    registry.save_data("/v1/images/" + layerid + "/layer",
+                                       datafile)
                     createdFiles.append(datafile)
 
             index = {
@@ -275,108 +385,6 @@ class DockerSource(base.Source):
                 except:
                     pass
 
-    def _save_data(self, template, server, path,
-                   dest, checksum=None):
-        try:
-            res = self._get_url(template, server, path)
-
-            datalen = res.info().getheader("Content-Length")
-            if datalen is not None:
-                datalen = int(datalen)
-
-            csum = None
-            if checksum is not None:
-                csum = hashlib.sha256()
-
-            pattern = [".", "o", "O", "o"]
-            patternIndex = 0
-            donelen = 0
-
-            with open(dest, "w") as f:
-                while 1:
-                    buf = res.read(1024*64)
-                    if not buf:
-                        break
-                    if csum is not None:
-                        csum.update(buf)
-                    f.write(buf)
-
-                    if datalen is not None:
-                        donelen = donelen + len(buf)
-                        debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
-                            pattern[patternIndex], (donelen/1024), (datalen/1024)
-                        ))
-                        patternIndex = (patternIndex + 1) % 4
-
-            debug("\x1b[K")
-            if csum is not None:
-                csumstr = "sha256:" + csum.hexdigest()
-                if csumstr != checksum:
-                    debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
-                    os.remove(dest)
-                    raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
-            debug("OK\n")
-            return res
-        except Exception, e:
-            debug("FAIL %s\n" % str(e))
-            raise
-
-    def _get_url(self, template, server, path, headers=None):
-        if template.protocol is None:
-            protocol = "https"
-        else:
-            protocol = template.protocol
-
-        if server is None:
-            if template.hostname is None:
-                server = "index.docker.io"
-            else:
-                if template.port is not None:
-                    server = "%s:%d" % (template.hostname, template.port)
-                else:
-                    server = template.hostname
-
-        url = urlparse.urlunparse((protocol, server, path, None, None, None))
-        debug("Fetching %s..." % url)
-
-        req = urllib2.Request(url=url)
-        if headers is not None:
-            for h in headers.keys():
-                req.add_header(h, headers[h])
-
-        self.auth_handler.prepare_req(req)
-
-        try:
-            res = urllib2.urlopen(req)
-            self.auth_handler.process_res(res)
-            return res
-        except urllib2.HTTPError as e:
-            if e.code == 401:
-                retry = self.auth_handler.process_err(e)
-                if retry:
-                    debug("Re-Fetching %s..." % url)
-                    self.auth_handler.prepare_req(req)
-                    res = urllib2.urlopen(req)
-                    self.auth_handler.process_res(res)
-                    return res
-                else:
-                    debug("Not re-fetching")
-                    raise
-            else:
-                raise
-
-    def _get_json(self, template, server, path):
-        try:
-            headers = {}
-            headers["Accept"] = "application/json")
-            res = self._get_url(template, server, path, headers)
-            data = json.loads(res.read())
-            debug("OK\n")
-            return (data, res)
-        except Exception, e:
-            debug("FAIL %s\n" % str(e))
-            raise
-
     def create_template(self, template, templatedir, connect=None):
         image = DockerImage.from_template(template)
 
-- 
2.7.4

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list



[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]