Introduce a class to handle HTTP requests with a docker registry
server, and associated auth credentials.
Signed-off-by: Daniel P. Berrange <berrange@xxxxxxxxxx>
---
libvirt-sandbox/image/sources/docker.py | 264 ++++++++++++++++----------------
1 file changed, 136 insertions(+), 128 deletions(-)
diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py
index a54f563..b2706b1 100644
--- a/libvirt-sandbox/image/sources/docker.py
+++ b/libvirt-sandbox/image/sources/docker.py
@@ -31,6 +31,7 @@ import shutil
import urlparse
import hashlib
from abc import ABCMeta, abstractmethod
+import copy
from . import base
@@ -152,14 +153,132 @@ class DockerAuthToken(DockerAuth):
return False
-class DockerSource(base.Source):
+class DockerRegistry():
- def __init__(self):
+ def __init__(self, uri_base):
+
+ self.uri_base = list(urlparse.urlparse(uri_base))
self.auth_handler = DockerAuthNop()
def set_auth_handler(self, auth_handler):
self.auth_handler = auth_handler
+ def set_server(self, server):
+ self.uri_base[1] = server
+
+ @classmethod
+ def from_template(cls, template):
+ protocol = template.protocol
+ hostname = template.hostname
+ port = template.port
+
+ if protocol is None:
+ protocol = "https"
+ if hostname is None:
+ hostname = "index.docker.io"
+
+ if port is None:
+ server = hostname
+ else:
+ server = "%s:%s" % (hostname, port)
+
+ url = urlparse.urlunparse((protocol, server, "", None, None, None))
+
+ return cls(url)
+
+ def get_url(self, path, headers=None):
+ url_bits = copy.copy(self.uri_base)
+ url_bits[2] = path
+ url = urlparse.urlunparse(url_bits)
+ debug("Fetching %s..." % url)
+
+ req = urllib2.Request(url=url)
+
+ if headers is not None:
+ for h in headers.keys():
+ req.add_header(h, headers[h])
+
+ self.auth_handler.prepare_req(req)
+
+ try:
+ res = urllib2.urlopen(req)
+ self.auth_handler.process_res(res)
+ return res
+ except urllib2.HTTPError as e:
+ if e.code == 401:
+ retry = self.auth_handler.process_err(e)
+ if retry:
+ debug("Re-Fetching %s..." % url)
+ self.auth_handler.prepare_req(req)
+ res = urllib2.urlopen(req)
+ self.auth_handler.process_res(res)
+ return res
+ else:
+ debug("Not re-fetching")
+ raise
+ else:
+ raise
+
+ def save_data(self, path, dest, checksum=None):
+ try:
+ res = self.get_url(path)
+
+ datalen = res.info().getheader("Content-Length")
+ if datalen is not None:
+ datalen = int(datalen)
+
+ csum = None
+ if checksum is not None:
+ csum = hashlib.sha256()
+
+ pattern = [".", "o", "O", "o"]
+ patternIndex = 0
+ donelen = 0
+
+ with open(dest, "w") as f:
+ while 1:
+ buf = res.read(1024*64)
+ if not buf:
+ break
+ if csum is not None:
+ csum.update(buf)
+ f.write(buf)
+
+ if datalen is not None:
+ donelen = donelen + len(buf)
+ debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
+ pattern[patternIndex], (donelen/1024), (datalen/1024)
+ ))
+ patternIndex = (patternIndex + 1) % 4
+
+ debug("\x1b[K")
+ if csum is not None:
+ csumstr = "sha256:" + csum.hexdigest()
+ if csumstr != checksum:
+ debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
+ os.remove(dest)
+ raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
+ debug("OK\n")
+ return res
+ except Exception, e:
+ debug("FAIL %s\n" % str(e))
+ raise
+
+ def get_json(self, path):
+ try:
+ headers = {}
+ headers["Accept"] = "application/json"
+ res = self.get_url(path, headers)
+ data = json.loads(res.read())
+ debug("OK\n")
+ return (data, res)
+ except Exception, e:
+ debug("FAIL %s\n" % str(e))
+ raise
+
+
+class DockerSource(base.Source):
+
def _check_cert_validate(self):
major = sys.version_info.major
SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
@@ -189,38 +308,33 @@ class DockerSource(base.Source):
def download_template(self, image, template, templatedir):
self._check_cert_validate()
+ registry = DockerRegistry.from_template(template)
basicauth = DockerAuthBasic(template.username, template.password)
- self.set_auth_handler(basicauth)
+ registry.set_auth_handler(basicauth)
try:
- (data, res) = self._get_json(template,
- None,
- "/v1/repositories/%s/%s/images" % (
- image.repo, image.name,
- ))
+ (data, res) = registry.get_json("/v1/repositories/%s/%s/images" % (
+ image.repo, image.name,
+ ))
except urllib2.HTTPError, e:
raise ValueError(["Image '%s' does not exist" % template])
registryendpoint = res.info().getheader('X-Docker-Endpoints')
if basicauth.token is not None:
- self.set_auth_handler(DockerAuthToken(basicauth.token))
+ registry.set_auth_handler(DockerAuthToken(basicauth.token))
else:
- self.set_auth_handler(DockerAuthNop())
+ registry.set_auth_handler(DockerAuthNop())
- (data, res) = self._get_json(template,
- registryendpoint,
- "/v1/repositories/%s/%s/tags" %(
- image.repo, image.name
- ))
+ (data, res) = registry.get_json("/v1/repositories/%s/%s/tags" %(
+ image.repo, image.name
+ ))
if image.tag not in data:
raise ValueError(["Tag '%s' does not exist for image '%s'" %
(image.tag, template)])
imagetagid = data[image.tag]
- (data, res) = self._get_json(template,
- registryendpoint,
- "/v1/images/" + imagetagid + "/ancestry")
+ (data, res) = registry.get_json("/v1/images/" + imagetagid + "/ancestry")
if data[0] != imagetagid:
raise ValueError(["Expected first layer id '%s' to match image id '%s'",
@@ -240,16 +354,12 @@ class DockerSource(base.Source):
datafile = layerdir + "/template.tar.gz"
if not os.path.exists(jsonfile) or not os.path.exists(datafile):
- res = self._save_data(template,
- registryendpoint,
- "/v1/images/" + layerid + "/json",
- jsonfile)
+ res = registry.save_data("/v1/images/" + layerid + "/json",
+ jsonfile)
createdFiles.append(jsonfile)
- self._save_data(template,
- registryendpoint,
- "/v1/images/" + layerid + "/layer",
- datafile)
+ registry.save_data("/v1/images/" + layerid + "/layer",
+ datafile)
createdFiles.append(datafile)
index = {
@@ -275,108 +385,6 @@ class DockerSource(base.Source):
except:
pass
- def _save_data(self, template, server, path,
- dest, checksum=None):
- try:
- res = self._get_url(template, server, path)
-
- datalen = res.info().getheader("Content-Length")
- if datalen is not None:
- datalen = int(datalen)
-
- csum = None
- if checksum is not None:
- csum = hashlib.sha256()
-
- pattern = [".", "o", "O", "o"]
- patternIndex = 0
- donelen = 0
-
- with open(dest, "w") as f:
- while 1:
- buf = res.read(1024*64)
- if not buf:
- break
- if csum is not None:
- csum.update(buf)
- f.write(buf)
-
- if datalen is not None:
- donelen = donelen + len(buf)
- debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
- pattern[patternIndex], (donelen/1024), (datalen/1024)
- ))
- patternIndex = (patternIndex + 1) % 4
-
- debug("\x1b[K")
- if csum is not None:
- csumstr = "sha256:" + csum.hexdigest()
- if csumstr != checksum:
- debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
- os.remove(dest)
- raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
- debug("OK\n")
- return res
- except Exception, e:
- debug("FAIL %s\n" % str(e))
- raise
-
- def _get_url(self, template, server, path, headers=None):
- if template.protocol is None:
- protocol = "https"
- else:
- protocol = template.protocol
-
- if server is None:
- if template.hostname is None:
- server = "index.docker.io"
- else:
- if template.port is not None:
- server = "%s:%d" % (template.hostname, template.port)
- else:
- server = template.hostname
-
- url = urlparse.urlunparse((protocol, server, path, None, None, None))
- debug("Fetching %s..." % url)
-
- req = urllib2.Request(url=url)
- if headers is not None:
- for h in headers.keys():
- req.add_header(h, headers[h])
-
- self.auth_handler.prepare_req(req)
-
- try:
- res = urllib2.urlopen(req)
- self.auth_handler.process_res(res)
- return res
- except urllib2.HTTPError as e:
- if e.code == 401:
- retry = self.auth_handler.process_err(e)
- if retry:
- debug("Re-Fetching %s..." % url)
- self.auth_handler.prepare_req(req)
- res = urllib2.urlopen(req)
- self.auth_handler.process_res(res)
- return res
- else:
- debug("Not re-fetching")
- raise
- else:
- raise
-
- def _get_json(self, template, server, path):
- try:
- headers = {}
- headers["Accept"] = "application/json")
- res = self._get_url(template, server, path, headers)
- data = json.loads(res.read())
- debug("OK\n")
- return (data, res)
- except Exception, e:
- debug("FAIL %s\n" % str(e))
- raise
-
def create_template(self, template, templatedir, connect=None):
image = DockerImage.from_template(template)
--
2.7.4
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list