Hey guys,
So I got bored today while waiting for some Satellites to install and
came up with this nifty way to group delegation paths together to reduce
the amount of XMLRPC calls made over the network while using
delegation. If this code works as well as it does on my box, we may
come close to meeting the performance ends we were hoping that delegated
func calls could provide. Anyways, here it is:
From 7f524383cc7f975545ff51e8719c0bffdbd114e7 Mon Sep 17 00:00:00 2001
From: Steve Salevan <ssalevan@xxxxxxxxxxxxxxxxxxxxxxxx>
Date: Wed, 16 Jul 2008 16:58:45 -0400
Subject: [PATCH] Adding delegation call grouping to reduce the amount of
necessary delegated calls to perform an action
---
func/minion/modules/delegation.py | 27 ++++++++++++++------
func/overlord/client.py | 18 ++++++++++----
func/overlord/delegation_tools.py | 48
++++++++++++++++++++++++++++++++++++-
3 files changed, 79 insertions(+), 14 deletions(-)
diff --git a/func/minion/modules/delegation.py
b/func/minion/modules/delegation.py
index 9f34d1a..38aef2c 100644
--- a/func/minion/modules/delegation.py
+++ b/func/minion/modules/delegation.py
@@ -11,6 +11,7 @@
import func_module
import func.overlord.client as fc
from func import utils
+from func.overlord import delegation_tools as dtools
class DelegationModule(func_module.FuncModule):
@@ -18,18 +19,28 @@ class DelegationModule(func_module.FuncModule):
api_version = "0.0.1"
description = "Minion-side module to support delegation on
sub-Overlords."
- def run(self,module,method,args,delegation_path):
+ def run(self,module,method,args,delegation_list):
"""
Delegates commands down the path of delegation
supplied as an argument
"""
+ result_dict = {}
- next_hop = delegation_path[0]
- overlord = fc.Overlord(next_hop)
- if len(delegation_path) == 1: #minion exists under this overlord
+ #separate list passed to us into minions we can call directly and
+ #further delegation paths
+ (single_paths, grouped_paths) = dtools.group_paths(delegation_list)
+
+ #run delegated calls
+ for group in grouped_paths.keys():
+ overlord = fc.Overlord(group)
+ path_list = grouped_paths[group]
+ delegation_results =
overlord.delegation.run(module,method,args,path_list)
+ result_dict.update(delegation_results[group]) #strip away
nesting hash
+
+ #run direct calls
+ for minion in single_paths:
+ overlord = fc.Overlord(minion)
overlord_module = getattr(overlord,module)
- return getattr(overlord_module,method)(*args[:])
+ result_dict.update(getattr(overlord_module,method)(*args[:]))
- stripped_list = delegation_path[1:len(delegation_path)]
- delegation_results =
overlord.delegation.run(module,method,args,stripped_list)
- return delegation_results[next_hop] #strip away nested hash
data from results
+ return result_dict
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 4cb8216..e29e875 100755
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -259,17 +259,24 @@ class Overlord(object):
return self.run_direct(module, method, args, nforks)
resulthash = {}
+ grouped_paths = []
#First we get all call paths for minions not directly beneath
this overlord
dele_paths = dtools.get_paths_for_glob(self.server_spec,
self.minionmap)
- non_single_paths = [path for path in dele_paths if len(path) > 1]
- for path in non_single_paths:
+ #Then we group them together in a dictionary by a common next hop
+ (single_paths,grouped_paths) = dtools.group_paths(dele_paths)
+ #print "single:%s" % single_paths
+ #print "grouped:%s" % grouped_paths
+
+ for group in grouped_paths.keys():
+ #print "mygroup:%s" % group
resulthash.update(self.run_direct(module,
method,
args,
nforks,
- call_path=path))
+
call_path=grouped_paths[group],
+ suboverlord=group))
#Next, we run everything that can be run directly beneath this
overlord
#Why do we do this after delegation calls? Imagine what
happens when
@@ -333,6 +340,7 @@ class Overlord(object):
if self.interactive:
print retval
+
except Exception, e:
(t, v, tb) = sys.exc_info()
retval = utils.nice_exception(t,v,tb)
@@ -349,10 +357,10 @@ class Overlord(object):
return (server_name, retval)
if kwargs.has_key('call_path'): #we're delegating if this key
exists
- spec = kwargs['call_path'][0] #the sub-overlord directly
beneath this one
+ delegation_path = kwargs['call_path']
+ spec = kwargs['suboverlord'] #the sub-overlord directly
beneath this one
minionobj = Minions(spec, port=self.port, verbose=self.verbose)
use_delegate = True #signal to process_server to call
delegate method
- delegation_path =
kwargs['call_path'][1:len(kwargs['call_path'])]
minionurls = minionobj.get_urls() #the single-item url list
to make async
#tools such as
jobthing/forkbomb happy
else: #we're directly calling minions, so treat everything normally
diff --git a/func/overlord/delegation_tools.py
b/func/overlord/delegation_tools.py
index f3445c0..7d0bc4f 100644
--- a/func/overlord/delegation_tools.py
+++ b/func/overlord/delegation_tools.py
@@ -17,6 +17,49 @@
import fnmatch
+class groupby(object):
+ """
+ Borrowing the groupby iterator class directly
+ from the Python API as it does not exist in Pythons < 2.4
+ """
+
+ def __init__(self, iterable, key=None):
+ if key is None:
+ key = lambda x: x
+ self.keyfunc = key
+ self.it = iter(iterable)
+ self.tgtkey = self.currkey = self.currvalue = xrange(0)
+ def __iter__(self):
+ return self
+ def next(self):
+ while self.currkey == self.tgtkey:
+ self.currvalue = self.it.next() # Exit on StopIteration
+ self.currkey = self.keyfunc(self.currvalue)
+ self.tgtkey = self.currkey
+ return (self.currkey, self._grouper(self.tgtkey))
+ def _grouper(self, tgtkey):
+ while self.currkey == tgtkey:
+ yield self.currvalue
+ self.currvalue = self.it.next() # Exit on StopIteration
+ self.currkey = self.keyfunc(self.currvalue)
+
+def group_paths(ungrouped_list):
+ """
+ Given a list of multi-element path lists,
+ groups them together into a list of single-element paths (which
+ exist directly under the current overlord) and a dictionary of paths
+ to send to next hops in the delegation chain, containing a list of
lists
+ keyed by their common next hop.
+ """
+
+ single_paths = [path[0] for path in ungrouped_list if len(path) == 1]
+ non_single_paths = [path for path in ungrouped_list if len(path) > 1]
+ path_group = dict([(key,[path[1:len(path)] for path in list(gen)])
+ for key, gen in groupby(non_single_paths,
+ key=lambda x:x[0])])
+
+ return (single_paths,path_group)
+
def get_paths_for_glob(glob, minionmap):
"""
Given a glob, returns shortest path to all minions
@@ -151,5 +194,8 @@ if __name__ == "__main__":
print "Element: %s, best path: %s" % (elem,
get_shortest_path(elem,mymap))
print "- And finally, with all duplicates removed:"
- for elem in get_paths_for_glob('*path*',mymap):
+ for elem in get_paths_for_glob('*',mymap):
print "Valid Path: %s" % elem
+
+ print "- And grouped together:"
+ print group_paths(get_paths_for_glob('*',mymap))
--
1.5.5.1
_______________________________________________
Func-list mailing list
Func-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/func-list
