[PATCH] - add puppet minion host cache file config option - add code to use that so we don't have to parse the host inventory + serials every time we load up a new instance

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



---
 func/commonconfig.py    |    1 +
 func/overlord/client.py |  112 ++++++++++++++++++++++++++++++-----------------
 2 files changed, 73 insertions(+), 40 deletions(-)

diff --git a/func/commonconfig.py b/func/commonconfig.py
index 93dfdc4..639e721 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -49,6 +49,7 @@ class OverlordConfig(BaseConfig):
     puppet_minions = BoolOption(False)
     puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
     puppet_signed_certs_dir = Option('/var/lib/puppet/ssl/ca/signed')
+    puppet_minion_cache = Option('/var/lib/func/minion_cache')
     puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
     host_down_list = Option('/var/lib/func/hosts_down.lst')
     allow_unknown_minions = BoolOption(False)
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 3fd2fc2..8d2accd 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -16,6 +16,7 @@ from func.jobthing import RETAIN_INTERVAL
 
 import sys
 import os
+from stat import *
 import time
 import shlex
 import subprocess
@@ -313,7 +314,7 @@ class PuppetMinions(Minions):
                  just_fqdns=False, groups_backend="conf",
                  delegate=False, minionmap={},exclude_spec=None,**kwargs):
         # local host_inv cache
-        self._host_inv = {}
+        self._cached_hosts = set([])
         self._revoked_serials = []
 
         Minions.__init__(self, spec, port=port, noglobs=noglobs, verbose=verbose,
@@ -328,8 +329,29 @@ class PuppetMinions(Minions):
         #these will be returned
         tmp_certs = set()
         tmp_hosts = set()
-        if not self._host_inv:
-            # get all hosts
+        
+        # if our cache file of hostnames is newer than both the inventory
+        # and the crl, then just read in that list of hostnames to a set
+        # and skip all of the below
+        if not self._cached_hosts:
+            cacheage = 0
+            invage = 1
+            crlage = 1
+            if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.exists(self.overlord_config.puppet_minion_cache):
+                cacheage = os.stat(self.overlord_config.puppet_minion_cache)[ST_MTIME]
+            if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.exists(self.overlord_config.puppet_inventory):
+                invage = os.stat(self.overlord_config.puppet_inventory)[ST_MTIME]
+            if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.exists(self.overlord_config.puppet_crl):
+                crlage = os.stat(self.overlord_config.puppet_crl)[ST_MTIME]
+            if cacheage >= invage and cacheage >= crlage:
+                self._cached_hosts.update(set(open(self.overlord_config.puppet_minion_cache, 'r').readlines()))
+                
+        if not self._cached_hosts:
+            # overview
+            # get all hosts from the puppet inventory
+            # remove all the hosts which are revoked by serial
+            # write out this list as a cache so we can load it quickly
+            # in the future if nothing else has changed.
             if os.access(self.overlord_config.puppet_inventory, os.R_OK):
                 fo = open(self.overlord_config.puppet_inventory, 'r')
                 host_inv = {}
@@ -356,48 +378,58 @@ class PuppetMinions(Minions):
                             continue
                     host_inv[hn] = serial
                 fo.close()
-                self._host_inv = host_inv # store ours
-
-        # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
-        if self.delegate:
-            try:
-                each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
-                shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
-            except IndexError:
-                return tmp_hosts,tmp_certs
-            else:
-                each_gloob = shortest_path[0]
-
-        # revoked certs
-        self._return_revoked_serials(self.overlord_config.puppet_crl)
-        for hostname in self._host_inv.keys():
-            if int(self._host_inv[hostname], 16) in self._revoked_serials:
-                continue
-            pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
-            if not os.path.exists(pempath):
-                continue
-            matched_gloob = False
-            if fnmatch.fnmatch(hostname, each_gloob):
-                matched_gloob = True
-                tmp_hosts.add(hostname)
-
-            # if we can't match this gloob and the gloob is not REALLY a glob
-            # then toss this at gethostbyname_ex() and see if any of the cname
-            # or aliases matches _something_ we know about
-            if not matched_gloob and not func_utils.re_glob(each_gloob):
-                found_by_alias = False
-                aliases = func_utils.get_all_host_aliases(each_gloob)
-                for name in aliases:
-                    if name in self._host_inv and int(self._host_inv[name], 16) not in self._revoked_serials:
-                        if os.path.exists(self.overlord_config.puppet_signed_certs_dir + '/' + name + '.pem'):
+                # don't include the ones which are of revoked certs
+                self._return_revoked_serials(self.overlord_config.puppet_crl)
+                for hostname in host_inv.keys():
+                    if int(host_inv[hostname], 16) in self._revoked_serials:
+                        continue
+                    pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
+                    if not os.path.exists(pempath):
+                        continue
+                    self._cached_hosts.add(hostname)
+                
+                # write out the cache of hosts minus the ones excluded by 
+                # revoked serials
+                if os.access(self.overlord_config.puppet_minion_cache, os.W_OK):
+                    hostcache = open(self.overlord_config.puppet_minion_cache, 'w')
+                    for host in self._cached_hosts:
+                        hostcache.write('%s\n' % host)
+                    hostcache.close()
+                    
+                
+               
+            # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
+            if self.delegate:
+                try:
+                    each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
+                    shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
+                except IndexError:
+                    return tmp_hosts,tmp_certs
+                else:
+                    each_gloob = shortest_path[0]
+
+            for hostname in self._cached_hosts:
+                matched_gloob = False
+                if fnmatch.fnmatch(hostname, each_gloob):
+                    matched_gloob = True
+                    tmp_hosts.add(hostname)
+
+                # if we can't match this gloob and the gloob is not REALLY a glob
+                # then toss this at gethostbyname_ex() and see if any of the cname
+                # or aliases matches _something_ we know about
+                if not matched_gloob and not func_utils.re_glob(each_gloob):
+                    found_by_alias = False
+                    aliases = func_utils.get_all_host_aliases(each_gloob)
+                    for name in aliases:
+                        if name in self._cached_hosts:
                             tmp_hosts.add(name)
                             found_by_alias = True
                             break
 
-                if self.overlord_config.allow_unknown_minions and not found_by_alias:
-                    tmp_hosts.add(each_gloob)
+                    if self.overlord_config.allow_unknown_minions and not found_by_alias:
+                        tmp_hosts.add(each_gloob)
 
-                # don't return certs path - just hosts
+                    # don't return certs path - just hosts
 
         return tmp_hosts,tmp_certs
 
-- 
1.7.4.4

_______________________________________________
Func-list mailing list
Func-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/func-list


[Index of Archives]     [Fedora Users]     [Linux Networking]     [Fedora Legacy List]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]

  Powered by Linux