--KFztAG8eRSV9hGtP
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Folks,
Attached is my failover patch again against the current 4_2 branch of
yum CVS. The differences this time are 2 things: 1) I made roundRobin
into priority (cause that's what it was) and made a real roundRobin
failover class. 2) I tweaked config.py to add the proper option to the
failovermethod key,
Jack
--
Jack Neely <slack@xxxxxxxxxxxxxxx>
Linux Realm Kit Administration and Development
PAMS Computer Operations at NC State University
GPG Fingerprint: 1917 5AC1 E828 9337 7AA4 EA6B 213B 765F 3B6A 5B89
--KFztAG8eRSV9hGtP
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="failover.patch"
? archwork.pyc
? callback.pyc
? clientStuff.pyc
? comps.pyc
? config.pyc
? failover.patch
? failover.py
? failover.pyc
? i18n.pyc
? keepalive.pyc
? logger.pyc
? nevral.pyc
? pkgaction.pyc
? rpmUtils.pyc
? urlgrabber.pyc
? yum
? yumcomps.pyc
? yumlock.pyc
? yummain.pyc
Index: clientStuff.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/clientStuff.py,v
retrieving revision 1.47.2.45
diff -u -r1.47.2.45 clientStuff.py
--- clientStuff.py 13 May 2003 05:24:03 -0000 1.47.2.45
+++ clientStuff.py 15 May 2003 21:18:03 -0000
@@ -599,7 +599,7 @@
if not conf.cache:
log(3, 'getting groups from server: %s' % serverid)
try:
- localgroupfile = retrygrab(remotegroupfile, localgroupfile, copy_local=1)
+ localgroupfile = grab(serverid, remotegroupfile, localgroupfile, copy_local=1)
except URLGrabError, e:
log(3, 'Error getting file %s' % remotegroupfile)
log(3, '%s' % e)
@@ -634,7 +634,7 @@
if not conf.cache:
log(3, 'Getting header.info from server')
try:
- headerinfofn = retrygrab(serverheader, localheaderinfo, copy_local=1)
+ headerinfofn = grab(serverid, serverheader, localheaderinfo, copy_local=1)
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % serverheader)
errorlog(0, '%s' % e)
@@ -657,7 +657,8 @@
for (name, arch) in nulist:
LocalHeaderFile = HeaderInfo.localHdrPath(name, arch)
RemoteHeaderFile = HeaderInfo.remoteHdrUrl(name, arch)
-
+ serverid = HeaderInfo.serverid(name, arch)
+
# if we have one cached, check it, if it fails, unlink it and continue
# as if it never existed
# else move along
@@ -681,8 +682,8 @@
if not conf.cache:
log(2, 'getting %s' % LocalHeaderFile)
try:
- hdrfn = retrygrab(RemoteHeaderFile, LocalHeaderFile, copy_local=1,
- checkfunc=(rpmUtils.checkheader, (name, arch), {}))
+ hdrfn = grab(serverid, RemoteHeaderFile, LocalHeaderFile, copy_local=1,
+ checkfunc=(rpmUtils.checkheader, (name, arch), {}))
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % RemoteHeaderFile)
errorlog(0, '%s' % e)
@@ -843,7 +844,7 @@
else:
log(2, 'Getting %s' % (os.path.basename(rpmloc)))
try:
- localrpmpath = retrygrab(tsInfo.remoteRpmUrl(name, arch), rpmloc, copy_local=0)
+ localrpmpath = grab(serverid, tsInfo.remoteRpmUrl(name, arch), rpmloc, copy_local=0)
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % tsInfo.remoteRpmUrl(name, arch))
errorlog(0, '%s' % e)
@@ -922,3 +923,50 @@
else:
size = size / 1000000000.0
return "%.2f GB" % size
+
+
+def grab(serverID, url, filename=None, copy_local=0, close_connection=0,
+ progress_obj=None, throttle=None, bandwidth=None,
+ numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None):
+
+ """Wrap retry grab and add in failover stuff. This needs access to
+ the conf class as well as the serverID.
+
+ We do look at retrycodes here to see if we should return or failover.
+ On fail we will raise the last exception that we got."""
+
+ fc = conf.get_failClass(serverID)
+ base = ''
+ for root in conf.serverurl[serverID]:
+ if string.find(url, root) == 0:
+ # We found the current base this url is made of
+ base = root
+ break
+ if base == '':
+ # We didn't find the base...something is wrong
+ raise Exception, "%s isn't made from a base URL I know about" % url
+ url = url[len(base):]
+ log(3, "failover: baseURL = " + base)
+ log(3, "failover: path = " + url)
+
+ # don't trust the base that the user supplied
+ base = fc.get_serverurl()
+ while base != None:
+ # Loop over baseURLs until one works or all are dead
+ try:
+ # XXX need some urlparse magic to keep this from breaking
+ return retrygrab(base+url, filename, copy_local,
+ close_connection, progress_obj, throttle,
+ bandwidth, numtries, retrycodes, checkfunc)
+ # What? We were successful?
+ except URLGrabError, e:
+ if e.errno in retrycodes:
+ log(2, "retrygrab() failed -- executing failover method")
+ fc.server_failed()
+ base = fc.get_serverurl()
+ if base == None:
+ log(1, "failover: out of servers to try")
+ raise
+ else:
+ raise
+
Index: config.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/config.py,v
retrieving revision 1.14.2.13
diff -u -r1.14.2.13 config.py
--- config.py 11 May 2003 06:04:03 -0000 1.14.2.13
+++ config.py 15 May 2003 21:18:03 -0000
@@ -23,6 +23,7 @@
import urllib
import rpm
import re
+import failover
import archwork
import rpmUtils
@@ -53,8 +54,9 @@
self.serverpkgdir = {}
self.serverhdrdir = {}
self.servercache = {}
- self.servergpgcheck= {}
- self.excludes= []
+ self.servergpgcheck={}
+ self.failoverclass = {}
+ self.excludes=[]
#defaults
self.cachedir = '/var/cache/yum'
@@ -117,23 +119,33 @@
for section in self.cfg.sections(): # loop through the list of sections
if section != 'main': # must be a serverid
name = self._getoption(section,'name')
- url = self._getoption(section,'baseurl')
- if name != None and url != None:
+ urls = self._doreplace(self._getoption(section,'baseurl'))
+ url = string.split(urls, ' ')
+ if name != None and url != []:
self.servers.append(section)
name = self._doreplace(name)
- url = self._doreplace(url)
self.servername[section] = name
self.serverurl[section] = url
+ failmeth = self._getoption(section,'failovermethod')
+ if failmeth == 'roundrobin':
+ failclass = failover.roundRobin(self, section)
+ elif failmeth == 'priority':
+ failclass = failover.priority(self, section)
+ else:
+ failclass = failover.roundRobin(self, section)
+ self.failoverclass[section] = failclass
if self._getoption(section,'gpgcheck') != None:
self.servergpgcheck[section]=self.cfg.getboolean(section,'gpgcheck')
else:
self.servergpgcheck[section]=0
-
- (s,b,p,q,f,o) = urlparse.urlparse(self.serverurl[section])
- # currently only allowing http, ftp and file url types
- if s not in ['http', 'ftp', 'file']:
- print _('Not using ftp, http or file for servers, Aborting - %s') % (self.serverurl[section])
- sys.exit(1)
+
+ for url in self.serverurl[section]:
+ (s,b,p,q,f,o) = urlparse.urlparse(url)
+ # currently only allowing http and ftp servers
+ if s not in ['http', 'ftp', 'file', 'https']:
+ print _('using ftp, http[s], or file for servers, Aborting - %s') % (url)
+ sys.exit(1)
+
cache = os.path.join(self.cachedir,section)
pkgdir = os.path.join(cache, 'packages')
hdrdir = os.path.join(cache, 'headers')
@@ -155,13 +167,19 @@
return None
def remoteGroups(self, serverid):
- return os.path.join(self.serverurl[serverid], 'yumgroups.xml')
+ return os.path.join(self.baseURL(serverid), 'yumgroups.xml')
def localGroups(self, serverid):
return os.path.join(self.servercache[serverid], 'yumgroups.xml')
def baseURL(self, serverid):
- return self.serverurl[serverid]
+ return self.get_failClass(serverid).get_serverurl()
+
+ def server_failed(self, serverid):
+ self.failoverclass[serverid].server_failed()
+
+ def get_failClass(self, serverid):
+ return self.failoverclass[serverid]
def remoteHeader(self, serverid):
return os.path.join(self.baseURL(serverid), 'headers/header.info')
Index: nevral.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/nevral.py,v
retrieving revision 1.28.2.24
diff -u -r1.28.2.24 nevral.py
--- nevral.py 6 May 2003 03:00:05 -0000 1.28.2.24
+++ nevral.py 15 May 2003 21:18:03 -0000
@@ -144,7 +144,7 @@
if l == 'in_rpm_db':
return l
hdrfn = self.hdrfn(name,arch)
- base = conf.serverurl[i]
+ base = conf.baseURL(i)
return base + '/headers/' + hdrfn
def localHdrPath(self, name, arch=None):
@@ -175,7 +175,7 @@
return None
if l == 'in_rpm_db':
return l
- base = conf.serverurl[i]
+ base = conf.baseURL(i)
return base +'/'+ l
def localRpmPath(self, name, arch=None):
--KFztAG8eRSV9hGtP
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="failover.py"
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2003 Jack Neely, NC State University
# Here we define a base class for failover methods. The idea here is that each
# failover method uses a class derived from the base class so yum only has to
# worry about calling get_serverurl() and server_failed() and these classes will
# figure out which URL to cough up based on the failover method.
import random
class baseFailOverMethod:
def __init__(self, conf, serverID):
# the yum conf structure
self.conf = conf
self.serverID = serverID
self.failures = 0
def get_serverurl(self):
"Returns a serverurl based on this failover method or None if complete failure."
return None
def server_failed(self):
"Tells the failover method that the current server is failed."
self.failures = self.failures + 1
def reset(self):
"Reset the failures counter."
self.failures = 0
class priority(baseFailOverMethod):
"""Chooses server based on the first success in the list."""
def get_serverurl(self):
"Returns a serverurl based on this failover method or None if complete failure."
if self.failures >= len(self.conf.serverurl[self.serverID]):
return None
return self.conf.serverurl[self.serverID][self.failures]
class roundRobin(baseFailOverMethod):
"""Chooses server based on a round robin."""
def __init__(self, conf, serverID):
baseFailOverMethod.__init__(self, conf, serverID)
random.seed()
self.offset = random.randint(0, 37)
def get_serverurl(self):
"Returns a serverurl based on this failover method or None if complete failure."
if self.failures >= len(self.conf.serverurl[self.serverID]):
return None
i = (self.failures + self.offset) % len(self.conf.serverurl[self.serverID])
return self.conf.serverurl[self.serverID][i]
--KFztAG8eRSV9hGtP--
