--UlVJffcvxoiEqYs2
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Folks,
I want failover.
I make yum smart.
This patch makes baseurl in the config file into a list. Uses spaces as
delimiters. Also adds a "failovermethod" to each server section to
select the failover method. If you leave it out it does round
robin...which is not really round robin right now...but yeah... Your
config files will work without making any changes...this just adds a few
extras options to them.
Attached are two things....a diff against cvs of the 4_2 branch and the
failover module I added. The basic idea is that we have a base failover
method class that gets inherited by other classes so folks can define
their own failover methods....sence I've seen lots of ideas suggested.
Mainly I need to touch up the failover classes but it works as a proof
of concept. Note that clientStuff has its own grab() function now.
go-go-gadget asbestos
--
Jack Neely <slack@xxxxxxxxxxxxxxx>
Linux Realm Kit Administration and Development
PAMS Computer Operations at NC State University
GPG Fingerprint: 1917 5AC1 E828 9337 7AA4 EA6B 213B 765F 3B6A 5B89
--UlVJffcvxoiEqYs2
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="failover.patch"
? archwork.pyc
? callback.pyc
? clientStuff.pyc
? comps.pyc
? config.pyc
? failover.patch
? failover.py
? failover.pyc
? i18n.pyc
? keepalive.pyc
? logger.pyc
? nevral.pyc
? pkgaction.pyc
? rpmUtils.pyc
? urlgrabber.pyc
? yum
? yumcomps.pyc
? yumlock.pyc
? yummain.pyc
Index: clientStuff.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/clientStuff.py,v
retrieving revision 1.47.2.45
diff -u -r1.47.2.45 clientStuff.py
--- clientStuff.py 13 May 2003 05:24:03 -0000 1.47.2.45
+++ clientStuff.py 14 May 2003 02:32:21 -0000
@@ -599,7 +599,7 @@
if not conf.cache:
log(3, 'getting groups from server: %s' % serverid)
try:
- localgroupfile = retrygrab(remotegroupfile, localgroupfile, copy_local=1)
+ localgroupfile = grab(serverid, remotegroupfile, localgroupfile, copy_local=1)
except URLGrabError, e:
log(3, 'Error getting file %s' % remotegroupfile)
log(3, '%s' % e)
@@ -634,7 +634,7 @@
if not conf.cache:
log(3, 'Getting header.info from server')
try:
- headerinfofn = retrygrab(serverheader, localheaderinfo, copy_local=1)
+ headerinfofn = grab(serverid, serverheader, localheaderinfo, copy_local=1)
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % serverheader)
errorlog(0, '%s' % e)
@@ -657,7 +657,8 @@
for (name, arch) in nulist:
LocalHeaderFile = HeaderInfo.localHdrPath(name, arch)
RemoteHeaderFile = HeaderInfo.remoteHdrUrl(name, arch)
-
+ serverid = HeaderInfo.serverid(name, arch)
+
# if we have one cached, check it, if it fails, unlink it and continue
# as if it never existed
# else move along
@@ -681,8 +682,8 @@
if not conf.cache:
log(2, 'getting %s' % LocalHeaderFile)
try:
- hdrfn = retrygrab(RemoteHeaderFile, LocalHeaderFile, copy_local=1,
- checkfunc=(rpmUtils.checkheader, (name, arch), {}))
+ hdrfn = grab(serverid, RemoteHeaderFile, LocalHeaderFile, copy_local=1,
+ checkfunc=(rpmUtils.checkheader, (name, arch), {}))
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % RemoteHeaderFile)
errorlog(0, '%s' % e)
@@ -843,7 +844,7 @@
else:
log(2, 'Getting %s' % (os.path.basename(rpmloc)))
try:
- localrpmpath = retrygrab(tsInfo.remoteRpmUrl(name, arch), rpmloc, copy_local=0)
+ localrpmpath = grab(serverid, tsInfo.remoteRpmUrl(name, arch), rpmloc, copy_local=0)
except URLGrabError, e:
errorlog(0, 'Error getting file %s' % tsInfo.remoteRpmUrl(name, arch))
errorlog(0, '%s' % e)
@@ -922,3 +923,50 @@
else:
size = size / 1000000000.0
return "%.2f GB" % size
+
+
+def grab(serverID, url, filename=None, copy_local=0, close_connection=0,
+ progress_obj=None, throttle=None, bandwidth=None,
+ numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None):
+
+ """Wrap retry grab and add in failover stuff. This needs access to
+ the conf class as well as the serverID.
+
+ We do look at retrycodes here to see if we should return or failover.
+ On fail we will raise the last exception that we got."""
+
+ fc = conf.get_failClass(serverID)
+ base = ''
+ for root in conf.serverurl[serverID]:
+ if string.find(url, root) == 0:
+ # We found the current base this url is made of
+ base = root
+ break
+ if base == '':
+ # We didn't find the base...something is wrong
+ raise Exception, "%s isn't made from a base URL I know about" % url
+ url = url[len(base):]
+ log(3, "failover: baseURL = " + base)
+ log(3, "failover: path = " + url)
+
+ # don't trust the base that the user supplied
+ base = fc.get_serverurl()
+ while base != None:
+ # Loop over baseURLs until one works or all are dead
+ try:
+ # XXX need some urlparse magic to keep this from breaking
+ return retrygrab(base+url, filename, copy_local,
+ close_connection, progress_obj, throttle,
+ bandwidth, numtries, retrycodes, checkfunc)
+ # What? We were successful?
+ except URLGrabError, e:
+ if e.errno in retrycodes:
+ log(2, "retrygrab() failed -- executing failover method")
+ fc.server_failed()
+ base = fc.get_serverurl()
+ if base == None:
+ log(1, "failover: out of servers to try")
+ raise
+ else:
+ raise
+
Index: config.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/config.py,v
retrieving revision 1.14.2.13
diff -u -r1.14.2.13 config.py
--- config.py 11 May 2003 06:04:03 -0000 1.14.2.13
+++ config.py 14 May 2003 02:32:21 -0000
@@ -23,6 +23,7 @@
import urllib
import rpm
import re
+import failover
import archwork
import rpmUtils
@@ -53,8 +54,9 @@
self.serverpkgdir = {}
self.serverhdrdir = {}
self.servercache = {}
- self.servergpgcheck= {}
- self.excludes= []
+ self.servergpgcheck={}
+ self.failoverclass = {}
+ self.excludes=[]
#defaults
self.cachedir = '/var/cache/yum'
@@ -117,23 +119,31 @@
for section in self.cfg.sections(): # loop through the list of sections
if section != 'main': # must be a serverid
name = self._getoption(section,'name')
- url = self._getoption(section,'baseurl')
- if name != None and url != None:
+ urls = self._doreplace(self._getoption(section,'baseurl'))
+ url = string.split(urls, ' ')
+ if name != None and url != []:
self.servers.append(section)
name = self._doreplace(name)
- url = self._doreplace(url)
self.servername[section] = name
self.serverurl[section] = url
+ failmeth = self._getoption(section,'failovermethod')
+ if failmeth == 'roundrobin':
+ failclass = failover.roundRobin(self, section)
+ else:
+ failclass = failover.roundRobin(self, section)
+ self.failoverclass[section] = failclass
if self._getoption(section,'gpgcheck') != None:
self.servergpgcheck[section]=self.cfg.getboolean(section,'gpgcheck')
else:
self.servergpgcheck[section]=0
-
- (s,b,p,q,f,o) = urlparse.urlparse(self.serverurl[section])
- # currently only allowing http, ftp and file url types
- if s not in ['http', 'ftp', 'file']:
- print _('Not using ftp, http or file for servers, Aborting - %s') % (self.serverurl[section])
- sys.exit(1)
+
+ for url in self.serverurl[section]:
+ (s,b,p,q,f,o) = urlparse.urlparse(url)
+ # currently only allowing http and ftp servers
+ if s not in ['http', 'ftp', 'file', 'https']:
+ print _('using ftp, http[s], or file for servers, Aborting - %s') % (url)
+ sys.exit(1)
+
cache = os.path.join(self.cachedir,section)
pkgdir = os.path.join(cache, 'packages')
hdrdir = os.path.join(cache, 'headers')
@@ -155,13 +165,19 @@
return None
def remoteGroups(self, serverid):
- return os.path.join(self.serverurl[serverid], 'yumgroups.xml')
+ return os.path.join(self.baseURL(serverid), 'yumgroups.xml')
def localGroups(self, serverid):
return os.path.join(self.servercache[serverid], 'yumgroups.xml')
def baseURL(self, serverid):
- return self.serverurl[serverid]
+ return self.get_failClass(serverid).get_serverurl()
+
+ def server_failed(self, serverid):
+ self.failoverclass[serverid].server_failed()
+
+ def get_failClass(self, serverid):
+ return self.failoverclass[serverid]
def remoteHeader(self, serverid):
return os.path.join(self.baseURL(serverid), 'headers/header.info')
Index: nevral.py
===================================================================
RCS file: /home//groups/yum/cvs/yum/nevral.py,v
retrieving revision 1.28.2.24
diff -u -r1.28.2.24 nevral.py
--- nevral.py 6 May 2003 03:00:05 -0000 1.28.2.24
+++ nevral.py 14 May 2003 02:32:21 -0000
@@ -144,7 +144,7 @@
if l == 'in_rpm_db':
return l
hdrfn = self.hdrfn(name,arch)
- base = conf.serverurl[i]
+ base = conf.baseURL(i)
return base + '/headers/' + hdrfn
def localHdrPath(self, name, arch=None):
@@ -175,7 +175,7 @@
return None
if l == 'in_rpm_db':
return l
- base = conf.serverurl[i]
+ base = conf.baseURL(i)
return base +'/'+ l
def localRpmPath(self, name, arch=None):
--UlVJffcvxoiEqYs2
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="failover.py"
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2003 Jack Neely, NC State University
# Here we define a base class for failover methods. The idea here is that each
# failover method uses a class derived from the base class so yum only has to
# worry about calling get_serverurl() and server_failed() and these classes will
# figure out which URL to cough up based on the failover method.
class baseFailOverMethod:
def __init__(self, conf, serverID):
# the yum conf structure
self.conf = conf
self.serverID = serverID
self.failures = 0
def get_serverurl(self):
"Returns a serverurl based on this failover method or None if complete failure."
return None
def server_failed(self):
"Tells the failover method that the current server is failed."
self.failures = self.failures + 1
def reset(self):
"Reset the failures counter."
self.failures = 0
class roundRobin(baseFailOverMethod):
"""Chooses server based on a round robin. The first in the list is always
first."""
def get_serverurl(self):
"Returns a serverurl based on this failover method or None if complete failure."
if self.failures >= len(self.conf.serverurl[self.serverID]):
return None
return self.conf.serverurl[self.serverID][self.failures]
--UlVJffcvxoiEqYs2--
