2 commits - func/commonconfig.py func/overlord

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



 func/commonconfig.py    |    1 
 func/overlord/client.py |   68 +++++++++++++++++++++++++++++++++++-------------
 2 files changed, 51 insertions(+), 18 deletions(-)

New commits:
commit 9ead0b646fd141973c9d871e3a6f771d961691be
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date:   Tue Jun 28 15:27:47 2011 -0400

    testing of this code found a number of errors, not least of
    which was one indentation too many for when the cache is populated.
    
    On my system of 142 puppet minions a '*' list_minions goes from 1.4-1.5s per load to
    0.2s per load

diff --git a/func/overlord/client.py b/func/overlord/client.py
index 8d2accd..b53dfdb 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -337,14 +337,14 @@ class PuppetMinions(Minions):
             cacheage = 0
             invage = 1
             crlage = 1
-            if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.exists(self.overlord_config.puppet_minion_cache):
+            if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.path.exists(self.overlord_config.puppet_minion_cache):
                 cacheage = os.stat(self.overlord_config.puppet_minion_cache)[ST_MTIME]
-            if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.exists(self.overlord_config.puppet_inventory):
+            if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.path.exists(self.overlord_config.puppet_inventory):
                 invage = os.stat(self.overlord_config.puppet_inventory)[ST_MTIME]
-            if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.exists(self.overlord_config.puppet_crl):
+            if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.path.exists(self.overlord_config.puppet_crl):
                 crlage = os.stat(self.overlord_config.puppet_crl)[ST_MTIME]
             if cacheage >= invage and cacheage >= crlage:
-                self._cached_hosts.update(set(open(self.overlord_config.puppet_minion_cache, 'r').readlines()))
+                self._cached_hosts.update(set([ line.strip() for line in open(self.overlord_config.puppet_minion_cache, 'r').readlines()]))
                 
         if not self._cached_hosts:
             # overview
@@ -390,7 +390,7 @@ class PuppetMinions(Minions):
                 
                 # write out the cache of hosts minus the ones excluded by 
                 # revoked serials
-                if os.access(self.overlord_config.puppet_minion_cache, os.W_OK):
+                if os.access(os.path.dirname(self.overlord_config.puppet_minion_cache), os.W_OK):
                     hostcache = open(self.overlord_config.puppet_minion_cache, 'w')
                     for host in self._cached_hosts:
                         hostcache.write('%s\n' % host)
@@ -398,38 +398,38 @@ class PuppetMinions(Minions):
                     
                 
                
-            # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
-            if self.delegate:
-                try:
-                    each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
-                    shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
-                except IndexError:
-                    return tmp_hosts,tmp_certs
-                else:
-                    each_gloob = shortest_path[0]
-
-            for hostname in self._cached_hosts:
-                matched_gloob = False
-                if fnmatch.fnmatch(hostname, each_gloob):
-                    matched_gloob = True
-                    tmp_hosts.add(hostname)
-
-                # if we can't match this gloob and the gloob is not REALLY a glob
-                # then toss this at gethostbyname_ex() and see if any of the cname
-                # or aliases matches _something_ we know about
-                if not matched_gloob and not func_utils.re_glob(each_gloob):
-                    found_by_alias = False
-                    aliases = func_utils.get_all_host_aliases(each_gloob)
-                    for name in aliases:
-                        if name in self._cached_hosts:
-                            tmp_hosts.add(name)
-                            found_by_alias = True
-                            break
-
-                    if self.overlord_config.allow_unknown_minions and not found_by_alias:
-                        tmp_hosts.add(each_gloob)
-
-                    # don't return certs path - just hosts
+        # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
+        if self.delegate:
+            try:
+                each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
+                shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
+            except IndexError:
+                return tmp_hosts,tmp_certs
+            else:
+                each_gloob = shortest_path[0]
+
+        for hostname in self._cached_hosts:
+            matched_gloob = False
+            if fnmatch.fnmatch(hostname, each_gloob):
+                matched_gloob = True
+                tmp_hosts.add(hostname)
+
+            # if we can't match this gloob and the gloob is not REALLY a glob
+            # then toss this at gethostbyname_ex() and see if any of the cname
+            # or aliases matches _something_ we know about
+            if not matched_gloob and not func_utils.re_glob(each_gloob):
+                found_by_alias = False
+                aliases = func_utils.get_all_host_aliases(each_gloob)
+                for name in aliases:
+                    if name in self._cached_hosts:
+                        tmp_hosts.add(name)
+                        found_by_alias = True
+                        break
+
+                if self.overlord_config.allow_unknown_minions and not found_by_alias:
+                    tmp_hosts.add(each_gloob)
+
+                # don't return certs path - just hosts
 
         return tmp_hosts,tmp_certs
 


commit 246f1311cff1ea341dc4a29c1325e75e4602c3ab
Author: Seth Vidal <skvidal@xxxxxxxxxxxxxxxxx>
Date:   Mon Jun 27 17:03:16 2011 -0400

    - add puppet minion host cache file config option
    - add code to use that so we don't have to parse the host inventory + serials
      every time we load up a new instance

diff --git a/func/commonconfig.py b/func/commonconfig.py
index 93dfdc4..639e721 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -49,6 +49,7 @@ class OverlordConfig(BaseConfig):
     puppet_minions = BoolOption(False)
     puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
     puppet_signed_certs_dir = Option('/var/lib/puppet/ssl/ca/signed')
+    puppet_minion_cache = Option('/var/lib/func/minion_cache')
     puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
     host_down_list = Option('/var/lib/func/hosts_down.lst')
     allow_unknown_minions = BoolOption(False)
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 3fd2fc2..8d2accd 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -16,6 +16,7 @@ from func.jobthing import RETAIN_INTERVAL
 
 import sys
 import os
+from stat import *
 import time
 import shlex
 import subprocess
@@ -313,7 +314,7 @@ class PuppetMinions(Minions):
                  just_fqdns=False, groups_backend="conf",
                  delegate=False, minionmap={},exclude_spec=None,**kwargs):
         # local host_inv cache
-        self._host_inv = {}
+        self._cached_hosts = set([])
         self._revoked_serials = []
 
         Minions.__init__(self, spec, port=port, noglobs=noglobs, verbose=verbose,
@@ -328,8 +329,29 @@ class PuppetMinions(Minions):
         #these will be returned
         tmp_certs = set()
         tmp_hosts = set()
-        if not self._host_inv:
-            # get all hosts
+        
+        # if our cache file of hostnames is newer than both the inventory
+        # and the crl, then just read in that list of hostnames to a set
+        # and skip all of the below
+        if not self._cached_hosts:
+            cacheage = 0
+            invage = 1
+            crlage = 1
+            if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.exists(self.overlord_config.puppet_minion_cache):
+                cacheage = os.stat(self.overlord_config.puppet_minion_cache)[ST_MTIME]
+            if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.exists(self.overlord_config.puppet_inventory):
+                invage = os.stat(self.overlord_config.puppet_inventory)[ST_MTIME]
+            if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.exists(self.overlord_config.puppet_crl):
+                crlage = os.stat(self.overlord_config.puppet_crl)[ST_MTIME]
+            if cacheage >= invage and cacheage >= crlage:
+                self._cached_hosts.update(set(open(self.overlord_config.puppet_minion_cache, 'r').readlines()))
+                
+        if not self._cached_hosts:
+            # overview
+            # get all hosts from the puppet inventory
+            # remove all the hosts which are revoked by serial
+            # write out this list as a cache so we can load it quickly
+            # in the future if nothing else has changed.
             if os.access(self.overlord_config.puppet_inventory, os.R_OK):
                 fo = open(self.overlord_config.puppet_inventory, 'r')
                 host_inv = {}
@@ -356,48 +378,58 @@ class PuppetMinions(Minions):
                             continue
                     host_inv[hn] = serial
                 fo.close()
-                self._host_inv = host_inv # store ours
-
-        # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
-        if self.delegate:
-            try:
-                each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
-                shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
-            except IndexError:
-                return tmp_hosts,tmp_certs
-            else:
-                each_gloob = shortest_path[0]
-
-        # revoked certs
-        self._return_revoked_serials(self.overlord_config.puppet_crl)
-        for hostname in self._host_inv.keys():
-            if int(self._host_inv[hostname], 16) in self._revoked_serials:
-                continue
-            pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
-            if not os.path.exists(pempath):
-                continue
-            matched_gloob = False
-            if fnmatch.fnmatch(hostname, each_gloob):
-                matched_gloob = True
-                tmp_hosts.add(hostname)
-
-            # if we can't match this gloob and the gloob is not REALLY a glob
-            # then toss this at gethostbyname_ex() and see if any of the cname
-            # or aliases matches _something_ we know about
-            if not matched_gloob and not func_utils.re_glob(each_gloob):
-                found_by_alias = False
-                aliases = func_utils.get_all_host_aliases(each_gloob)
-                for name in aliases:
-                    if name in self._host_inv and int(self._host_inv[name], 16) not in self._revoked_serials:
-                        if os.path.exists(self.overlord_config.puppet_signed_certs_dir + '/' + name + '.pem'):
+                # don't include the ones which are of revoked certs
+                self._return_revoked_serials(self.overlord_config.puppet_crl)
+                for hostname in host_inv.keys():
+                    if int(host_inv[hostname], 16) in self._revoked_serials:
+                        continue
+                    pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
+                    if not os.path.exists(pempath):
+                        continue
+                    self._cached_hosts.add(hostname)
+                
+                # write out the cache of hosts minus the ones excluded by 
+                # revoked serials
+                if os.access(self.overlord_config.puppet_minion_cache, os.W_OK):
+                    hostcache = open(self.overlord_config.puppet_minion_cache, 'w')
+                    for host in self._cached_hosts:
+                        hostcache.write('%s\n' % host)
+                    hostcache.close()
+                    
+                
+               
+            # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
+            if self.delegate:
+                try:
+                    each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
+                    shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
+                except IndexError:
+                    return tmp_hosts,tmp_certs
+                else:
+                    each_gloob = shortest_path[0]
+
+            for hostname in self._cached_hosts:
+                matched_gloob = False
+                if fnmatch.fnmatch(hostname, each_gloob):
+                    matched_gloob = True
+                    tmp_hosts.add(hostname)
+
+                # if we can't match this gloob and the gloob is not REALLY a glob
+                # then toss this at gethostbyname_ex() and see if any of the cname
+                # or aliases matches _something_ we know about
+                if not matched_gloob and not func_utils.re_glob(each_gloob):
+                    found_by_alias = False
+                    aliases = func_utils.get_all_host_aliases(each_gloob)
+                    for name in aliases:
+                        if name in self._cached_hosts:
                             tmp_hosts.add(name)
                             found_by_alias = True
                             break
 
-                if self.overlord_config.allow_unknown_minions and not found_by_alias:
-                    tmp_hosts.add(each_gloob)
+                    if self.overlord_config.allow_unknown_minions and not found_by_alias:
+                        tmp_hosts.add(each_gloob)
 
-                # don't return certs path - just hosts
+                    # don't return certs path - just hosts
 
         return tmp_hosts,tmp_certs
 


_______________________________________________
Func-list mailing list
Func-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/func-list


[Index of Archives]     [Fedora Users]     [Linux Networking]     [Fedora Legacy List]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]

  Powered by Linux