func/commonconfig.py | 16 ++
func/minion/server.py | 29 +++-
func/overlord/base_command.py | 8 -
func/overlord/client.py | 191 ++++++++++++++++++++++++++-----
func/overlord/cmd_modules/grep.py | 80 ++++++++----
func/overlord/cmd_modules/listminions.py | 4
func/overlord/command.py | 6
func/overlord/func_command.py | 10 +
func/overlord/groups.py | 4
9 files changed, 271 insertions(+), 77 deletions(-)
New commits:
commit af77bcb8c90a4d9e46bc4e9a6c616377bb70a6fc
Merge: 17eda83... e4038cc...
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Fri Mar 19 10:35:55 2010 -0400
Merge branch 'master' of ssh://git.fedorahosted.org/git/func
* 'master' of ssh://git.fedorahosted.org/git/func:
base64 encode all strings before transmitting over xmlrpc
commit 17eda83451b0ae7d72dfce0939666b67d4c21e7a
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Mon Mar 15 12:24:18 2010 -0400
fix wrong indentation :(
diff --git a/func/overlord/cmd_modules/grep.py b/func/overlord/cmd_modules/grep.py
index dc6bb5f..db60cd6 100644
--- a/func/overlord/cmd_modules/grep.py
+++ b/func/overlord/cmd_modules/grep.py
@@ -137,9 +137,9 @@ class Grep(base_command.BaseCommand):
module_methods = self.overlord_obj.system.inventory()
for hn in module_methods:
- if type(module_methods[hn]) != types.DictType:
- sys.stderr.write("Error on host %s: %s" % (hn, ' '.join(module_methods[hn])))
- continue
+ if type(module_methods[hn]) != types.DictType:
+ sys.stderr.write("Error on host %s: %s" % (hn, ' '.join(module_methods[hn])))
+ continue
for module in module_methods[hn]:
# searching for "grep"? meta
commit 3299661255943bcd9cd54ac4333824c99a8b93d4
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Wed Mar 10 16:47:08 2010 -0500
fixup puppet minions a bit
add a puppet_signed_certs_dir option to point to where puppet stores
the pem files. This is necessary in order to completely know about hosts
which were not revoked but were --cleaned from the puppetdb
also make sure we're looking for revoked serials only after sorting
for the latest serials per host.
diff --git a/func/commonconfig.py b/func/commonconfig.py
index f35d576..76607a4 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -44,6 +44,7 @@ class OverlordConfig(BaseConfig):
ca_file = Option('')
puppet_minions = BoolOption(False)
puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
+ puppet_signed_certs_dir = Option('/var/lib/puppet/ssl/ca/signed')
puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
host_down_list = Option('/var/lib/func/hosts_down.lst')
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 27a92c7..d92dce1 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -281,8 +281,6 @@ class PuppetMinions(Minions):
tmp_certs = set()
tmp_hosts = set()
- # revoked certs
- revoked_serials = self._return_revoked_serials(self.overlord_config.puppet_crl)
# get all hosts
if os.access(self.overlord_config.puppet_inventory, os.R_OK):
fo = open(self.overlord_config.puppet_inventory, 'r')
@@ -293,8 +291,6 @@ class PuppetMinions(Minions):
if re.match('\s*(#|$)', line):
continue
(serial, before, after, cn) = line.split()
- if int(serial, 16) in revoked_serials:
- continue
before = time.strftime('%s', time.strptime(before, time_format))
if now < int(before):
continue
@@ -308,8 +304,16 @@ class PuppetMinions(Minions):
if host_inv[hn] > serial:
continue
host_inv[hn] = serial
-
+ fo.close()
+
+ # revoked certs
+ revoked_serials = self._return_revoked_serials(self.overlord_config.puppet_crl)
for hostname in host_inv.keys():
+ if int(host_inv[hostname], 16) in revoked_serials:
+ continue
+ pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
+ if not os.path.exists(pempath):
+ continue
if fnmatch.fnmatch(hostname, each_gloob):
tmp_hosts.add(hostname)
# don't return certs path - just hosts
commit bcf9cba252bbe02e3840a457a0a64030cd82e4b8
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Tue Mar 9 13:26:54 2010 -0500
add a host_down_list - a flat (and optional) list of hosts which are known to be down
this keeps us from bothering to connect to hosts we know are out of commission right now.
works with both minion kinds
diff --git a/func/commonconfig.py b/func/commonconfig.py
index f8e2c7f..f35d576 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -45,4 +45,5 @@ class OverlordConfig(BaseConfig):
puppet_minions = BoolOption(False)
puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
-
+ host_down_list = Option('/var/lib/func/hosts_down.lst')
+
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 1b05964..27a92c7 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -107,6 +107,7 @@ class Minions(object):
self.exclude_spec = exclude_spec
self.cm_config = read_config(CONFIG_FILE, CMConfig)
+ self.overlord_config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
self.group_class = groups.Groups(backend=groups_backend,
get_hosts_for_spec=self.get_hosts_for_spec,
**kwargs)
@@ -115,7 +116,8 @@ class Minions(object):
self.all_hosts = set()
self.all_certs = set()
self.all_urls = []
-
+ self._downed_hosts = []
+
def _get_new_hosts(self):
self.new_hosts = self._get_group_hosts(self.spec)
return self.new_hosts
@@ -210,11 +212,14 @@ class Minions(object):
self._get_new_hosts()
self._get_all_hosts()
hosts = self.all_hosts
- results = self.all_urls
- else:
- results = []
-
+
+ results = []
+
for host in hosts:
+ if host in self.downed_hosts:
+ sys.stderr.write("%s excluded due to being listed in %s\n" % (host, self.overlord_config.host_down_list))
+ # FIXME maybe we should splat something to the logs?
+ continue
if not self.just_fqdns:
host_res = "https://%s:%s"; % (host, self.port)
else:
@@ -235,33 +240,38 @@ class Minions(object):
return True
return False
+ def _get_downed_hosts(self):
+ """returns a list of minions which are known to not be up"""
+ if self._downed_hosts:
+ return self._downed_hosts
+
+ hosts = []
+ if self.overlord_config.host_down_list and \
+ os.path.exists(self.overlord_config.host_down_list):
+ fo = open(self.overlord_config.host_down_list, 'r')
+ for line in fo.readlines():
+ if re.match('\s*(#|$)', line):
+ continue
+ hn = line.replace('\n','')
+ if hn not in hosts:
+ hosts.append(hn)
+ fo.close()
+
+ self._downed_hosts = hosts
+
+ return self._downed_hosts
+
+ downed_hosts = property(fget=lambda self: self._get_downed_hosts())
class PuppetMinions(Minions):
def __init__(self, spec, port=51234,
noglobs=None, verbose=None,
just_fqdns=False, groups_backend="conf",
delegate=False, minionmap={},exclude_spec=None,**kwargs):
- self.spec = spec
- self.port = port
- self.noglobs = noglobs
- self.verbose = verbose
- self.just_fqdns = just_fqdns
- self.delegate = delegate
- self.minionmap = minionmap
- self.exclude_spec = exclude_spec
-
- self.cm_config = read_config(CONFIG_FILE, CMConfig)
- self.overlord_config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
-
- self.group_class = groups.Groups(backend=groups_backend,
- get_hosts_for_spec=self.get_hosts_for_spec,
- **kwargs)
- #lets make them sets so we dont loop again and again
- self.all_hosts = set()
- self.all_certs = set()
- self.all_urls = []
-
-
+ Minions.__init__(self, spec, port=port, noglobs=noglobs, verbose=verbose,
+ just_fqdns=just_fqdns, groups_backend=groups_backend,
+ delegate=delegate, minionmap=minionmap,
+ exclude_spec=exclude_spec,**kwargs)
def _get_hosts_for_spec(self,each_gloob):
"""
commit cdb37a22316de3c89047a308fbe9843cfd2b8e5b
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Tue Mar 9 13:24:59 2010 -0500
make sure when we get back errors connecting to list grep modules
that we do something intelligent with them
diff --git a/func/overlord/cmd_modules/grep.py b/func/overlord/cmd_modules/grep.py
index b7f1ac5..dc6bb5f 100644
--- a/func/overlord/cmd_modules/grep.py
+++ b/func/overlord/cmd_modules/grep.py
@@ -15,6 +15,7 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import pprint
import sys
+import types
from func.overlord import client
from func.overlord import base_command
@@ -136,8 +137,12 @@ class Grep(base_command.BaseCommand):
module_methods = self.overlord_obj.system.inventory()
for hn in module_methods:
+ if type(module_methods[hn]) != types.DictType:
+ sys.stderr.write("Error on host %s: %s" % (hn, ' '.join(module_methods[hn])))
+ continue
+
for module in module_methods[hn]:
- # searching for "grep"? meta
+ # searching for "grep"? meta
if "grep" in module_methods[hn][module]:
if not host_modules.has_key(host):
host_modules[host] = []
commit f87aec595f9de95e40088e9f19ee85af51a72b89
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Fri Mar 5 10:14:43 2010 -0500
add in the option of using the openSSL.crypto.load_crl method if it is available
diff --git a/func/overlord/client.py b/func/overlord/client.py
index fdbc301..1b05964 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -307,18 +307,28 @@ class PuppetMinions(Minions):
return tmp_hosts,tmp_certs
def _return_revoked_serials(self, crlfile):
- call = '/usr/bin/openssl crl -text -noout -in %s' % crlfile
- call = shlex.split(call)
- serials = []
- (res,err) = subprocess.Popen(call, stdout=subprocess.PIPE).communicate()
- for line in res.split('\n'):
- if line.find('Serial Number:') == -1:
- continue
- (crap, serial) = line.split(':')
- serial = serial.strip()
- serial = int(serial, 16)
- serials.append(serial)
- return serials
+ try:
+ serials = []
+ crltext = open(crlfile, 'r').read()
+ from OpenSSL import crypto
+ crl = crypto.load_crl(crypto.FILETYPE_PEM, crltext)
+ revs = crl.get_revoked()
+ for revoked in revs:
+ serials.append(str(revoked.get_serial()))
+ return serials
+ except (ImportError, AttributeError), e:
+ call = '/usr/bin/openssl crl -text -noout -in %s' % crlfile
+ call = shlex.split(call)
+ serials = []
+ (res,err) = subprocess.Popen(call, stdout=subprocess.PIPE).communicate()
+ for line in res.split('\n'):
+ if line.find('Serial Number:') == -1:
+ continue
+ (crap, serial) = line.split(':')
+ serial = serial.strip()
+ serial = int(serial, 16)
+ serials.append(serial)
+ return serials
# does the hostnamegoo actually expand to anything?
commit c1d2da547e5357fa3bf2200b133ddffd61aca3d2
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Thu Mar 4 23:33:00 2010 -0500
rework how the overlord object and the minions object play together.
this lets us define a minions class by the overlord config and work
within that for additional host_specs.
- clean up PuppetMinions class
- make grep, listminions and groups stop importing Minions and overlord directly
- make grep, listminions, groups use the overlord objects with the proper
config settings, not just the defaults.
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 7e0d8fa..fdbc301 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -107,7 +107,9 @@ class Minions(object):
self.exclude_spec = exclude_spec
self.cm_config = read_config(CONFIG_FILE, CMConfig)
- self.group_class = groups.Groups(backend=groups_backend,**kwargs)
+ self.group_class = groups.Groups(backend=groups_backend,
+ get_hosts_for_spec=self.get_hosts_for_spec,
+ **kwargs)
#lets make them sets so we dont loop again and again
self.all_hosts = set()
@@ -203,19 +205,28 @@ class Minions(object):
#we keep it all the time as a set so
return list(self.all_hosts)
- def get_urls(self):
- self._get_new_hosts()
- self._get_all_hosts()
- for host in self.all_hosts:
+ def get_urls(self, hosts=[]):
+ if not hosts:
+ self._get_new_hosts()
+ self._get_all_hosts()
+ hosts = self.all_hosts
+ results = self.all_urls
+ else:
+ results = []
+
+ for host in hosts:
if not self.just_fqdns:
- self.all_urls.append("https://%s:%s"; % (host, self.port))
+ host_res = "https://%s:%s"; % (host, self.port)
else:
- self.all_urls.append(host)
+ host_res = host
+
+ if not host_res in results: # this might get slow if there are thousands of hosts
+ results.append(host_res)
- if self.verbose and len(self.all_urls) == 0:
+ if self.verbose and len(results) == 0:
sys.stderr.write("no hosts matched\n")
- return self.all_urls
+ return results
# FIXME: hmm, dont like this bit of the api... -al;
def is_minion(self):
@@ -225,17 +236,11 @@ class Minions(object):
return False
-class PuppetMinions(object):
- def __init__(self, spec, port=51234,
+class PuppetMinions(Minions):
+ def __init__(self, spec, port=51234,
noglobs=None, verbose=None,
just_fqdns=False, groups_backend="conf",
delegate=False, minionmap={},exclude_spec=None,**kwargs):
-
- # open inventory.txt
- # for each CN (uniqued) in there
- # open the ca_crl.pem file - if the serial of the CN shows up in there
- # remove those from the list of hosts
-
self.spec = spec
self.port = port
self.noglobs = noglobs
@@ -247,35 +252,16 @@ class PuppetMinions(object):
self.cm_config = read_config(CONFIG_FILE, CMConfig)
self.overlord_config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
- self.cm_config = read_config(CONFIG_FILE, CMConfig)
- self.group_class = groups.Groups(backend=groups_backend,**kwargs)
-
+
+ self.group_class = groups.Groups(backend=groups_backend,
+ get_hosts_for_spec=self.get_hosts_for_spec,
+ **kwargs)
#lets make them sets so we dont loop again and again
self.all_hosts = set()
self.all_certs = set()
self.all_urls = []
- def _get_new_hosts(self):
- self.new_hosts = self._get_group_hosts(self.spec)
- return self.new_hosts
- def _get_group_hosts(self,spec):
- return self.group_class.get_hosts_by_group_glob(spec)
-
- def _get_hosts_for_specs(self,seperate_gloobs):
- """
- Gets the hosts and certs for proper spec
- """
- tmp_hosts = set()
- tmp_certs = set()
- for each_gloob in seperate_gloobs:
- if each_gloob.startswith('@'):
- continue
- h,c = self._get_hosts_for_spec(each_gloob)
- tmp_hosts = tmp_hosts.union(h)
- tmp_certs = tmp_certs.union(c)
-
- return tmp_hosts,tmp_certs
def _get_hosts_for_spec(self,each_gloob):
"""
@@ -334,67 +320,6 @@ class PuppetMinions(object):
serials.append(serial)
return serials
- def get_hosts_for_spec(self,spec):
- """
- Be careful when editting that method it will be used
- also by groups api to pull machines to have better
- glob control there ...
- """
- return self._get_hosts_for_spec(spec)[0]
-
-
-
- def _get_all_hosts(self):
- """
- Gets hosts that are included and excluded by user
- a better orm like spec so user may say
- func "*" --exclude "www.*;@mygroup" ...
- """
- included_part = self._get_hosts_for_specs(self.spec.split(";")+self.new_hosts)
- self.all_certs=self.all_certs.union(included_part[1])
- self.all_hosts=self.all_hosts.union(included_part[0])
- #excluded ones
- if self.exclude_spec:
- #get first groups ypu dont want to run :
- group_exclude = self._get_group_hosts(self.exclude_spec)
- excluded_part = self._get_hosts_for_specs(self.exclude_spec.split(";")+group_exclude)
- self.all_certs = self.all_certs.difference(excluded_part[1])
- self.all_hosts = self.all_hosts.difference(excluded_part[0])
-
-
-
- def get_all_hosts(self):
- """
- Get current host list
- """
- self._get_new_hosts()
- self._get_all_hosts()
-
- #we keep it all the time as a set so
- return list(self.all_hosts)
-
- def get_urls(self):
- self._get_new_hosts()
- self._get_all_hosts()
- for host in self.all_hosts:
- if not self.just_fqdns:
- self.all_urls.append("https://%s:%s"; % (host, self.port))
- else:
- self.all_urls.append(host)
-
- if self.verbose and len(self.all_urls) == 0:
- sys.stderr.write("no hosts matched\n")
-
- return self.all_urls
-
- # FIXME: hmm, dont like this bit of the api... -al;
- def is_minion(self):
- self.get_urls()
- if len(self.all_urls) > 0:
- return True
- return False
-
-
# does the hostnamegoo actually expand to anything?
def is_minion(minion_string):
@@ -426,18 +351,17 @@ class Overlord(object):
@config -- optional config object
"""
- self.config = read_config(FUNCD_CONFIG_FILE, FuncdConfig)
+ self.cm_config = read_config(CONFIG_FILE, CMConfig)
+ self.funcd_config = read_config(FUNCD_CONFIG_FILE, FuncdConfig)
+ self.config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
+ if config:
+ self.config = config
- self.cm_config = config
- if config is None:
- self.cm_config = read_config(CONFIG_FILE, CMConfig)
-
- self.overlord_config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
+ self.overlord_config = self.config # for backward compat
self.server_spec = server_spec
self.exclude_spec = exclude_spec
- # we could make this settable in overlord.conf as well
self.port = port
if self.config.listen_port:
self.port = self.config.listen_port
@@ -449,8 +373,8 @@ class Overlord(object):
# the default
self.timeout = DEFAULT_TIMEOUT
# the config file
- if self.overlord_config.socket_timeout != 0.0:
- self.timeout = self.overlord_config.socket_timeout
+ if self.config.socket_timeout != 0.0:
+ self.timeout = self.config.socket_timeout
# commandline
if timeout:
self.timeout = timeout
@@ -466,15 +390,16 @@ class Overlord(object):
#overlord_query stuff
self.overlord_query = OverlordQuery()
- if self.overlord_config.puppet_minions:
- mc = PuppetMinions
+ if self.config.puppet_minions:
+ self._mc = PuppetMinions
else:
- mc = Minions
+ self._mc = Minions
- self.minions_class = mc(self.server_spec, port=self.port,
+ self.minions_class = self._mc(self.server_spec, port=self.port,
noglobs=self.noglobs, verbose=self.verbose,
exclude_spec=self.exclude_spec)
self.minions = self.minions_class.get_urls()
+
if len(self.minions) == 0:
raise Func_Client_Exception, 'Can\'t find any minions matching \"%s\". ' % self.server_spec
@@ -497,12 +422,12 @@ class Overlord(object):
# funcd key, cert, ca
# raise FuncClientError
- if not client_key and self.overlord_config.key_file != '':
- client_key = self.overlord_config.key_file
- if not client_cert and self.overlord_config.cert_file != '':
- client_cert = self.overlord_config.cert_file
- if not ca and self.overlord_config.ca_file != '':
- ca = self.overlord_config.ca_file
+ if not client_key and self.config.key_file != '':
+ client_key = self.config.key_file
+ if not client_cert and self.config.cert_file != '':
+ client_cert = self.config.cert_file
+ if not ca and self.config.ca_file != '':
+ ca = self.config.ca_file
ol_key = '%s/certmaster.key' % self.cm_config.cadir
ol_crt = '%s/certmaster.crt' % self.cm_config.cadir
@@ -798,7 +723,6 @@ class Overlord(object):
def process_server(bucketnumber, buckets, server):
-
conn = sslclient.FuncServer(server, self.key, self.cert, self.ca, self.timeout)
# conn = xmlrpclib.ServerProxy(server)
@@ -856,13 +780,14 @@ class Overlord(object):
if kwargs.has_key('call_path'): #we're delegating if this key exists
delegation_path = kwargs['call_path']
spec = kwargs['suboverlord'] #the sub-overlord directly beneath this one
- minionobj = Minions(spec, port=self.port, verbose=self.verbose)
+ minions_hosts = self.minions_class.get_hosts_for_spec(spec)
use_delegate = True #signal to process_server to call delegate method
- minionurls = minionobj.get_urls() #the single-item url list to make async
+ minionurls = minionobj.get_urls(hosts=minion_hosts) #the single-item url list to make async
#tools such as jobthing/forkbomb happy
else: #we're directly calling minions, so treat everything normally
spec = self.server_spec
- minionurls = self.minions
+ minionurls = self.minions_class.get_urls()
+
#print "Minion_url is :",minionurls
#print "Process server is :",process_server
@@ -870,7 +795,6 @@ class Overlord(object):
if self.nforks > 1 or self.async:
# using forkbomb module to distribute job over multiple threads
if not self.async:
-
results = forkbomb.batch_run(minionurls, process_server, nforks)
else:
minion_info =dict(spec=spec,module=module,method=method)
@@ -882,10 +806,11 @@ class Overlord(object):
(nkey,nvalue) = process_server(0, 0, x)
results[nkey] = nvalue
else:
+
# globbing is not being used, but still need to make sure
# URI is well formed.
# expanded = expand_servers(self.server_spec, port=self.port, noglobs=True, verbose=self.verbose)[0]
- expanded_minions = Minions(spec, port=self.port, noglobs=True, verbose=self.verbose)
+ expanded_minions = self._mc(spec, port=self.port, noglobs=True, verbose=self.verbose)
minions = expanded_minions.get_urls()[0]
results = process_server(0, 0, minions)
diff --git a/func/overlord/cmd_modules/grep.py b/func/overlord/cmd_modules/grep.py
index cade1d7..b7f1ac5 100644
--- a/func/overlord/cmd_modules/grep.py
+++ b/func/overlord/cmd_modules/grep.py
@@ -40,16 +40,15 @@ class Grep(base_command.BaseCommand):
default=self.delegate,
action="store_true")
self.parser.add_option('-m', '--modules', dest="modules",
- help="a list of modules to be searched",
- default=self.delegate,
- action="store",
- type="string")
+ help="modules to be searched",
+ default=[],
+ action="append")
def handleOptions(self, options):
self.options = options
self.verbose = options.verbose
-
+
# I'm not really a fan of the "module methodname" approach
# but we'll keep it for now -akl
@@ -84,28 +83,35 @@ class Grep(base_command.BaseCommand):
# which is better, this is across hosts (aka, walk across the
# hosts, then ask all the module.grep methods to it, then on to
# next host
-
+
+ existent_minions_class = self.overlord_obj.minions_class # keep a copy
+
for host in host_modules.keys():
+ host_only_mc = self.overlord_obj._mc(host, noglobs=True)
+ host_only_mc.get_all_hosts()
+ self.overlord_obj.minions_class = host_only_mc
for module in host_modules[host]:
- if self.options.verbose:
- print "Scanning module: %s on host: %s" %(module, host)
+ if self.options.modules and module in self.options.modules:
+ if self.options.verbose:
+ print "Scanning module: %s on host: %s" %(module, host)
+
+ tmp_res = self.overlord_obj.run(module,"grep",[self.word])
- tmp_res = self.overlord_obj.run(module,"grep",[self.word])
-
- if self.options.async:
- tmp_res = self.overlord_obj.local.utils.async_poll(tmp_res,None)
- #FIXME: I'm not sure what the best format for this is...
- if tmp_res[host]:
- print "%s: %s" % (host, pprint.pformat(tmp_res[host]))
-
+ if self.options.async:
+ tmp_res = self.overlord_obj.local.utils.async_poll(tmp_res,None)
+ #FIXME: I'm not sure what the best format for this is...
+ if tmp_res[host]:
+ print "%s: %s" % (host, pprint.pformat(tmp_res[host]))
+
+ self.overlord_obj.minions_class = existent_minions_class # put it back
+
def _get_host_grep_modules(self, server_spec):
"""
In cases when user doesnt supply the module list
we have to consider that all of the modules are
chosen so that method will return a list of them
"""
- from func.overlord.client import Minions,Overlord
-
+
#insetad of getting all of the modules we consider
#that all of machines has the same modules ...
@@ -114,19 +120,28 @@ class Grep(base_command.BaseCommand):
#FIXME: we need to change this to create a dict of hostname->modules
# so we only call module.grep on systems that report it. things like
# virt/hardware aren't available on all guests
- m = Minions(server_spec)
- hosts = m.get_all_hosts()
+
+ if not hasattr(self, 'overlord_obj'):
+ self.getOverlord()
+
+ hosts = self.overlord_obj.minions_class.get_all_hosts()
+ existent_minions_class = self.overlord_obj.minions_class # keep a copy
if not hosts:
raise Exception("No minions on system!")
+
+
for host in hosts:
- fc = Overlord(host, noglobs=True)
- module_methods = fc.system.inventory()
+ host_only_mc = self.overlord_obj._mc(host, noglobs=True)
+ self.overlord_obj.minions_class = host_only_mc
+ module_methods = self.overlord_obj.system.inventory()
- for module in module_methods:
+ for hn in module_methods:
+ for module in module_methods[hn]:
# searching for "grep"? meta
- if "grep" in module_methods[module]:
- if not host_modules.has_key(host):
- host_modules[host] = []
- host_modules[host].append(module)
-
+ if "grep" in module_methods[hn][module]:
+ if not host_modules.has_key(host):
+ host_modules[host] = []
+ host_modules[host].append(module)
+
+ self.overlord_obj.minions_class = existent_minions_class # put it back
return host_modules
diff --git a/func/overlord/cmd_modules/listminions.py b/func/overlord/cmd_modules/listminions.py
index 5264791..8d36d15 100644
--- a/func/overlord/cmd_modules/listminions.py
+++ b/func/overlord/cmd_modules/listminions.py
@@ -37,8 +37,8 @@ class ListMinions(base_command.BaseCommand):
def do(self, args):
self.server_spec = self.parentCommand.server_spec
-
- minion_set = client.Minions(self.server_spec, port=self.port)
+ self.getOverlord()
+ minion_set = self.overlord_obj.minions_class
servers = minion_set.get_all_hosts()
servers.sort()
for server in servers:
diff --git a/func/overlord/groups.py b/func/overlord/groups.py
index 80754f0..b957a63 100644
--- a/func/overlord/groups.py
+++ b/func/overlord/groups.py
@@ -20,6 +20,10 @@ class Groups(object):
Initialize the backend you are going to use
"""
#initialize here the backend
+ if 'get_hosts_for_spec' in kwargs:
+ self.get_hosts_for_spec = kwargs['get_hosts_for_spec']
+ del kwargs['get_hosts_for_spec']
+
self.backend = choose_backend(**kwargs)
commit 9ad4c99a332004a765e8ad4dd7071e0f02b06ed2
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Thu Mar 4 23:32:06 2010 -0500
add listen port to overlord config so there is a default port in
the overlord config that it will attempt to connect to minions on
diff --git a/func/commonconfig.py b/func/commonconfig.py
index 46c2216..f8e2c7f 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -36,6 +36,7 @@ class FuncdConfig(BaseConfig):
class OverlordConfig(BaseConfig):
socket_timeout = FloatOption(0)
+ listen_port = IntOption('51234')
backend = Option('conf')
group_db = Option('')
key_file = Option('')
commit cad967ae990b82ca115b821171b991a91622933a
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Thu Mar 4 23:30:34 2010 -0500
add support for a -c/--conf overlord.conf conf file option to the
func command. Also remove a bunch of unneeded imports from command
diff --git a/func/overlord/base_command.py b/func/overlord/base_command.py
index fd77e4b..275bd95 100644
--- a/func/overlord/base_command.py
+++ b/func/overlord/base_command.py
@@ -33,13 +33,16 @@ class BaseCommand(command.Command):
forks=1
delegate=False
mapfile=DEFAULT_MAPLOC
-
+
# temporary work around FIXME
# we really need a way to store what port each minion is
# listening on, though this is probably workable for most
# cases. Though it should probably be a different config
# file, since FuncdConfig is for the minion server, not
def getOverlord(self):
+ ol_config = None
+ if self.parentCommand.conffile:
+ ol_config = read_config(self.parentCommand.conffile, commonconfig.OverlordConfig)
self.overlord_obj = client.Overlord(self.server_spec,
interactive=self.interactive,
verbose=self.verbose,
@@ -48,6 +51,7 @@ class BaseCommand(command.Command):
delegate=self.delegate,
mapfile=self.mapfile,
timeout=self.parentCommand.socket_timeout,
- exclude_spec=self.parentCommand.exclude_spec)
+ exclude_spec=self.parentCommand.exclude_spec,
+ config=ol_config)
diff --git a/func/overlord/command.py b/func/overlord/command.py
index 0a8e1c6..1290728 100644
--- a/func/overlord/command.py
+++ b/func/overlord/command.py
@@ -14,9 +14,6 @@ Command class.
import optparse
import sys
-from certmaster.config import read_config, CONFIG_FILE
-
-from certmaster.commonconfig import CMConfig
class CommandHelpFormatter(optparse.IndentedHelpFormatter):
"""
@@ -124,8 +121,7 @@ class Command:
self.stdout = stdout
self.stderr = stderr
self.parentCommand = parentCommand
-
- self.config = read_config(CONFIG_FILE, CMConfig)
+
# create subcommands if we have them
self.subCommands = {}
diff --git a/func/overlord/func_command.py b/func/overlord/func_command.py
index 8f4dffa..a443693 100644
--- a/func/overlord/func_command.py
+++ b/func/overlord/func_command.py
@@ -26,7 +26,8 @@ class FuncCommandLine(command.Command):
socket_timeout = None
subCommandClasses = []
exclude_spec = None
-
+ conffile = None
+
def __init__(self):
modules = module_loader.load_modules('func/overlord/cmd_modules/', base_command.BaseCommand)
for x in modules.keys():
@@ -43,7 +44,8 @@ class FuncCommandLine(command.Command):
help="exclude some of minions",
action="store",
type="string")
-
+ self.parser.add_option('-c', '--conf', dest="conffile",
+ help="specify an overlord.conf file for func to use")
# just some ugly goo to try to guess if arg[1] is hostnamegoo or
# a command name
@@ -72,3 +74,7 @@ class FuncCommandLine(command.Command):
if options.exclude:
self.exclude_spec = options.exclude
+
+ if options.conffile:
+ self.conffile = options.conffile
+
commit 70356202f3e8d238992ac97248dd887d3e5eca94
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Tue Feb 23 11:59:02 2010 -0500
have funcd use our specified ca/cert/keys and if we have turned off use_certmaster don't
bother talking to the Certmaster on startup for a certificate, just assume we have it.
diff --git a/func/commonconfig.py b/func/commonconfig.py
index c71138d..46c2216 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -29,6 +29,7 @@ class FuncdConfig(BaseConfig):
minion_name = Option('')
method_log_dir = Option("/var/log/func/methods/")
+ use_certmaster = BoolOption(True)
ca_file = Option('')
cert_file = Option('')
key_file = Option('')
diff --git a/func/minion/server.py b/func/minion/server.py
index fbe5c4b..96f796f 100644
--- a/func/minion/server.py
+++ b/func/minion/server.py
@@ -100,6 +100,9 @@ class XmlRpcInterface(object):
methods.sort()
return methods
+ def load_module(self, name):
+ """FIXME load a module and set it up on the running xmlrpc instance"""
+ pass
import func.minion.modules.func_module as fm
@fm.findout
@@ -243,13 +246,23 @@ class FuncSSLXMLRPCServer(AuthedXMLRPCServer.AuthedSSLXMLRPCServer,
XmlRpcInterface.__init__(self)
hn = func_utils.get_hostname_by_route()
-
- self.key = "%s/%s.pem" % (self.cm_config.cert_dir, hn)
- self.cert = "%s/%s.cert" % (self.cm_config.cert_dir, hn)
- self.ca = "%s/ca.cert" % self.cm_config.cert_dir
+
+ if self.config.key_file != '':
+ self.key = self.config.key_file
+ else:
+ self.key = "%s/%s.pem" % (self.cm_config.cert_dir, hn)
+
+ if self.config.cert_file != '':
+ self.cert = self.config.cert_file
+ else:
+ self.cert = "%s/%s.cert" % (self.cm_config.cert_dir, hn)
+ if self.config.ca_file != '':
+ self.ca = self.config.ca_file
+ else:
+ self.ca = "%s/ca.cert" % self.cm_config.cert_dir
+
self._our_ca = certs.retrieve_cert_from_file(self.ca)
-
self.acls = acls_mod.Acls(config=self.config)
AuthedXMLRPCServer.AuthedSSLXMLRPCServer.__init__(self, args,
@@ -358,8 +371,10 @@ def main(argv):
print "serving...\n"
try:
- hn = futils.get_hostname_by_route()
- requester.request_cert(hn)
+ config = read_config("/etc/func/minion.conf", FuncdConfig)
+ if config.use_certmaster:
+ hn = futils.get_hostname_by_route()
+ requester.request_cert(hn)
serve()
except codes.FuncException, e:
print >> sys.stderr, 'error: %s' % e
commit 57b93fec75731d26fd6b9b18a6171a9cbe86eebb
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Tue Feb 23 11:58:36 2010 -0500
use our ca/cert/keys specified in our overlord config
diff --git a/func/overlord/client.py b/func/overlord/client.py
index b0631e7..7e0d8fa 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -496,6 +496,14 @@ class Overlord(object):
# certmaster key, cert, ca
# funcd key, cert, ca
# raise FuncClientError
+
+ if not client_key and self.overlord_config.key_file != '':
+ client_key = self.overlord_config.key_file
+ if not client_cert and self.overlord_config.cert_file != '':
+ client_cert = self.overlord_config.cert_file
+ if not ca and self.overlord_config.ca_file != '':
+ ca = self.overlord_config.ca_file
+
ol_key = '%s/certmaster.key' % self.cm_config.cadir
ol_crt = '%s/certmaster.crt' % self.cm_config.cadir
myname = func_utils.get_hostname_by_route()
commit 9a9f7c9f2f7c56e799a97d348abb2056ab6dca56
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date: Mon Feb 22 17:35:48 2010 -0500
config code changes and basic puppetminions class to have func get it's
host inventory and paths from puppet
diff --git a/func/commonconfig.py b/func/commonconfig.py
index 3823d39..c71138d 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -29,10 +29,18 @@ class FuncdConfig(BaseConfig):
minion_name = Option('')
method_log_dir = Option("/var/log/func/methods/")
+ ca_file = Option('')
+ cert_file = Option('')
+ key_file = Option('')
class OverlordConfig(BaseConfig):
socket_timeout = FloatOption(0)
backend = Option('conf')
group_db = Option('')
-
+ key_file = Option('')
+ cert_file = Option('')
+ ca_file = Option('')
+ puppet_minions = BoolOption(False)
+ puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
+ puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
diff --git a/func/overlord/client.py b/func/overlord/client.py
index e011929..b0631e7 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -18,6 +18,10 @@ import sys
import glob
import os
import time
+import shlex
+import subprocess
+import re
+import fnmatch
import func.yaml as yaml
from certmaster.commonconfig import CMConfig
@@ -221,6 +225,176 @@ class Minions(object):
return False
+class PuppetMinions(object):
+ def __init__(self, spec, port=51234,
+ noglobs=None, verbose=None,
+ just_fqdns=False, groups_backend="conf",
+ delegate=False, minionmap={},exclude_spec=None,**kwargs):
+
+ # open inventory.txt
+ # for each CN (uniqued) in there
+ # open the ca_crl.pem file - if the serial of the CN shows up in there
+ # remove those from the list of hosts
+
+ self.spec = spec
+ self.port = port
+ self.noglobs = noglobs
+ self.verbose = verbose
+ self.just_fqdns = just_fqdns
+ self.delegate = delegate
+ self.minionmap = minionmap
+ self.exclude_spec = exclude_spec
+
+ self.cm_config = read_config(CONFIG_FILE, CMConfig)
+ self.overlord_config = read_config(OVERLORD_CONFIG_FILE, OverlordConfig)
+ self.cm_config = read_config(CONFIG_FILE, CMConfig)
+ self.group_class = groups.Groups(backend=groups_backend,**kwargs)
+
+ #lets make them sets so we dont loop again and again
+ self.all_hosts = set()
+ self.all_certs = set()
+ self.all_urls = []
+
+ def _get_new_hosts(self):
+ self.new_hosts = self._get_group_hosts(self.spec)
+ return self.new_hosts
+
+ def _get_group_hosts(self,spec):
+ return self.group_class.get_hosts_by_group_glob(spec)
+
+ def _get_hosts_for_specs(self,seperate_gloobs):
+ """
+ Gets the hosts and certs for proper spec
+ """
+ tmp_hosts = set()
+ tmp_certs = set()
+ for each_gloob in seperate_gloobs:
+ if each_gloob.startswith('@'):
+ continue
+ h,c = self._get_hosts_for_spec(each_gloob)
+ tmp_hosts = tmp_hosts.union(h)
+ tmp_certs = tmp_certs.union(c)
+
+ return tmp_hosts,tmp_certs
+
+ def _get_hosts_for_spec(self,each_gloob):
+ """
+ Pull only for specified spec
+ """
+ #these will be returned
+ tmp_certs = set()
+ tmp_hosts = set()
+
+ # revoked certs
+ revoked_serials = self._return_revoked_serials(self.overlord_config.puppet_crl)
+ # get all hosts
+ if os.access(self.overlord_config.puppet_inventory, os.R_OK):
+ fo = open(self.overlord_config.puppet_inventory, 'r')
+ host_inv = {}
+ time_format = '%Y-%m-%dT%H:%M:%S%Z'
+ now = time.time()
+ for line in fo.readlines():
+ if re.match('\s*(#|$)', line):
+ continue
+ (serial, before, after, cn) = line.split()
+ if int(serial, 16) in revoked_serials:
+ continue
+ before = time.strftime('%s', time.strptime(before, time_format))
+ if now < int(before):
+ continue
+ after = time.strftime('%s', time.strptime(after, time_format))
+ if now > int(after):
+ continue
+
+ hn = cn.replace('/CN=','')
+ hn = hn.replace('\n','')
+ if hn in host_inv:
+ if host_inv[hn] > serial:
+ continue
+ host_inv[hn] = serial
+
+ for hostname in host_inv.keys():
+ if fnmatch.fnmatch(hostname, each_gloob):
+ tmp_hosts.add(hostname)
+ # don't return certs path - just hosts
+
+ return tmp_hosts,tmp_certs
+
+ def _return_revoked_serials(self, crlfile):
+ call = '/usr/bin/openssl crl -text -noout -in %s' % crlfile
+ call = shlex.split(call)
+ serials = []
+ (res,err) = subprocess.Popen(call, stdout=subprocess.PIPE).communicate()
+ for line in res.split('\n'):
+ if line.find('Serial Number:') == -1:
+ continue
+ (crap, serial) = line.split(':')
+ serial = serial.strip()
+ serial = int(serial, 16)
+ serials.append(serial)
+ return serials
+
+ def get_hosts_for_spec(self,spec):
+ """
+ Be careful when editting that method it will be used
+ also by groups api to pull machines to have better
+ glob control there ...
+ """
+ return self._get_hosts_for_spec(spec)[0]
+
+
+
+ def _get_all_hosts(self):
+ """
+ Gets hosts that are included and excluded by user
+ a better orm like spec so user may say
+ func "*" --exclude "www.*;@mygroup" ...
+ """
+ included_part = self._get_hosts_for_specs(self.spec.split(";")+self.new_hosts)
+ self.all_certs=self.all_certs.union(included_part[1])
+ self.all_hosts=self.all_hosts.union(included_part[0])
+ #excluded ones
+ if self.exclude_spec:
+ #get first groups ypu dont want to run :
+ group_exclude = self._get_group_hosts(self.exclude_spec)
+ excluded_part = self._get_hosts_for_specs(self.exclude_spec.split(";")+group_exclude)
+ self.all_certs = self.all_certs.difference(excluded_part[1])
+ self.all_hosts = self.all_hosts.difference(excluded_part[0])
+
+
+
+ def get_all_hosts(self):
+ """
+ Get current host list
+ """
+ self._get_new_hosts()
+ self._get_all_hosts()
+
+ #we keep it all the time as a set so
+ return list(self.all_hosts)
+
+ def get_urls(self):
+ self._get_new_hosts()
+ self._get_all_hosts()
+ for host in self.all_hosts:
+ if not self.just_fqdns:
+ self.all_urls.append("https://%s:%s"; % (host, self.port))
+ else:
+ self.all_urls.append(host)
+
+ if self.verbose and len(self.all_urls) == 0:
+ sys.stderr.write("no hosts matched\n")
+
+ return self.all_urls
+
+ # FIXME: hmm, dont like this bit of the api... -al;
+ def is_minion(self):
+ self.get_urls()
+ if len(self.all_urls) > 0:
+ return True
+ return False
+
+
# does the hostnamegoo actually expand to anything?
def is_minion(minion_string):
@@ -292,8 +466,14 @@ class Overlord(object):
#overlord_query stuff
self.overlord_query = OverlordQuery()
-
- self.minions_class = Minions(self.server_spec, port=self.port, noglobs=self.noglobs, verbose=self.verbose,exclude_spec=self.exclude_spec)
+ if self.overlord_config.puppet_minions:
+ mc = PuppetMinions
+ else:
+ mc = Minions
+
+ self.minions_class = mc(self.server_spec, port=self.port,
+ noglobs=self.noglobs, verbose=self.verbose,
+ exclude_spec=self.exclude_spec)
self.minions = self.minions_class.get_urls()
if len(self.minions) == 0:
raise Func_Client_Exception, 'Can\'t find any minions matching \"%s\". ' % self.server_spec
_______________________________________________
Func-list mailing list
Func-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/func-list
