Merge branch 'cache_lookup_api' into 'master'

Cache lookup api

See merge request nounous/ansible!18

ansible.cfg

@@ -45,3 +45,18 @@ api_hostname = intranet.crans.org
 
 # Whether or not using vault_cranspasswords
 use_cpasswords = True
+
+# Specify cache plugin for re2o API. By default, cache nothing
+cache = jsonfile
+
+# Only used for memcached plugin
+# List of connection information for the memcached DBs
+# Default is ['127.0.0.1:11211']
+# memcached_connection = ['127.0.0.1:11211']
+
+# Time in second before the cache expired. 0 means never expire cache.
+# Default is 24 hours.
+timeout = 86400
+
+# Default is 12 hours.
+timeout_token = 43200

lookup_plugins/re2oapi.py

@@ -7,6 +7,8 @@ For a detailed example look at https://github.com/ansible/ansible/blob/3dbf89e8a
 The API Client has been adapted from https://gitlab.federez.net/re2o/re2oapi
 """
 
+from ansible.plugins.loader import cache_loader
+
 from pathlib import Path
 import datetime
 import requests
@@ -28,38 +30,67 @@ from ansible.config.manager import ConfigManager
 # Ansible Logger to stdout
 display = Display()
 
-# Number of seconds before expiration where renewing the token is done
-TIME_FOR_RENEW = 120
 # Default name of the file to store tokens. Path $HOME/{DEFAUlt_TOKEN_FILENAME}
 DEFAULT_TOKEN_FILENAME = '.re2o.token'
 
+# If no plugin is used, then use this as token timeout.
+# Overriden by key timeout_token from ansible configuration.
+TIME_FOR_RENEW = 43200 # 12 jours
 
 class Client:
     """
     Class based client to contact re2o API.
     """
-    def __init__(self, hostname, username, password, use_tls=True):
+    def __init__(self, hostname, username, password,
+                 use_tls=True, cachetoken=None):
         """
         :arg hostname: The hostname of the Re2o instance to use.
         :arg username: The username to use.
         :arg password: The password to use.
         :arg use_tls: A boolean to specify whether the client should use a
                       a TLS connection. Default is True. Please, keep it.
+        :arg cachetoken: The cache to use to manage authentication token.
+                         If it is None, then store the token in a file.
         """
         self.use_tls = use_tls
         self.hostname = hostname
         self._username = username
         self._password = password
-
+        self._cachetoken = cachetoken
-        self.token_file = Path.home() / DEFAULT_TOKEN_FILENAME
+        self.token_file = None
+        if self._cachetoken is None:
+            self.token_file = Path.home() / DEFAULT_TOKEN_FILENAME
+            display.vvv("Setting token file to {}".format(self.token_file))
+        else:
+            try:
+                display.vvv("Using {} as cache plugin"
+                            .format(self._cachetoken.plugin_name))
+            except AttributeError:
+                # Happens when plugin_name is not implemented...
+                # For example with memcached
+                display.vvv("Using cache plugin specified in configuration.")
 
         display.v("Connecting to {hostname} as user {user}".format(
             hostname=to_native(self.hostname), user=to_native(self._username)))
-        try:
+
-            self.token = self._get_token_from_file()
+    @property
-        except AnsibleFileNotFound:
+    def token(self):
-            display.vv("Force renew the token")
+        if self._cachetoken:
-            self._force_renew_token()
+            display.vvv("Trying to get token from cache.")
+            if self._cachetoken.contains("auth_token"):
+                display.vvv("Found token in cache.")
+                return self._cachetoken.get("auth_token")
+            else:
+                display.vvv("Token not found. Forcing renew.")
+                return self._force_renew_token()
+        else:
+            try:
+                token = self._get_token_from_file()
+                if token['expiration'] < datetime.datetime.now() + \
+                   datetime.timedelta(seconds=TIME_FOR_RENEW):
+                    return self._force_renew_token()
+            except AnsibleError:
+                return self._force_renew_token()
 
     def _get_token_from_file(self):
         display.vv("Trying to fetch token from {}".format(self.token_file))
@@ -93,13 +124,18 @@ class Client:
                                               )
                                      )
         else:
-            display.vv("""Token successfully retreived from
+            display.vv("Token successfully retreived from "
-            file {token}""".format(token=self.token_file))
+                       "file {token}".format(token=self.token_file))
             return ret
 
     def _force_renew_token(self):
-        self.token = self._get_token_from_server()
+        token = self._get_token_from_server()
-        self._save_token_to_file()
+        if self._cachetoken:
+            display.vvv("Storing authentication token in cache")
+            self._cachetoken.set("auth_token", token.get('token'))
+        else:
+            self._save_token_to_file(token)
+        return token.get('token')
 
     def _get_token_from_server(self):
         display.vv("Requesting a new token for {user}@{host}".format(
@@ -139,7 +175,7 @@ class Client:
     def _parse_date(self, date, date_format="%Y-%m-%dT%H:%M:%S"):
         return datetime.datetime.strptime(date.split('.')[0], date_format)
 
-    def _save_token_to_file(self):
+    def _save_token_to_file(self, token):
         display.vv("Saving token to file {}".format(self.token_file))
         try:
             # Read previous data to avoid erasures
@@ -153,8 +189,8 @@ class Client:
         if self.hostname not in data.keys():
             data[self.hostname] = {}
         data[self.hostname][self._username] = {
-            'token': self.token['token'],
+            'token': token['token'],
-            'expiration': self.token['expiration'].isoformat(),
+            'expiration': token['expiration'].isoformat(),
         }
 
         try:
@@ -169,22 +205,6 @@ class Client:
             display.vv("Token successfully written to file {}"
                        .format(self.token_file))
 
-    def get_token(self):
-        """
-        Retrieves the token to use for the current connection.
-        Automatically renewed if needed.
-        """
-        if self.need_renew_token:
-            self._force_renew_token()
-
-        return self.token['token']
-
-    @property
-    def need_renew_token(self):
-        return self.token['expiration'] < \
-            datetime.datetime.now() + \
-            datetime.timedelta(seconds=TIME_FOR_RENEW)
-
     def _request(self, method, url, headers={}, params={}, *args, **kwargs):
         display.vv("Building the {method} request to {url}.".format(
             method=method.upper(),
@@ -192,9 +212,9 @@ class Client:
         ))
 
         # Force the 'Authorization' field with the right token.
-        display.vvv("Forcing authentication token.")
+        display.vvv("Forcing authentication token in headers.")
         headers.update({
-            'Authorization': 'Token {}'.format(self.get_token())
+            'Authorization': 'Token {}'.format(self.token)
         })
 
         # Use a json format unless the user already specified something
@@ -213,10 +233,10 @@ class Client:
             # Force re-login to the server (case of a wrong token but valid
             # credentials) and then retry the request without catching errors.
             display.vv("Token refused. Trying to refresh the token.")
-            self._force_renew_token()
+            token = self._force_renew_token()
 
             headers.update({
-                'Authorization': 'Token {}'.format(self.get_token())
+                'Authorization': 'Token {}'.format(token)
             })
             display.vv("Re-performing the request {method} {url}".format(
                 method=method.upper(),
@@ -340,6 +360,107 @@ class LookupModule(LookupBase):
         - debug: var=dnszones
     """
 
+    def _readconfig(self, section="re2o", key=None, default=None,
+                    boolean=False, integer=False):
+        config = self._config
+        if not config:
+            return default
+        else:
+            if config.has_option(section, key):
+                display.vvv("Found key {} in configuration file".format(key))
+                if boolean:
+                    return config.getboolean(section, key)
+                elif integer:
+                    return config.getint(section, key)
+                else:
+                    return config.get(section, key)
+            else:
+                return default
+
+    def _manage_cachedir(self, cachedir=None, plugin=None):
+        try:
+            self._uri = cachedir / plugin
+        except Exception:
+            raise AnsibleError("Undefined specification for cache plugin")
+
+        display.vvv("Cache directory is {}".format(self._uri))
+        if not self._uri.exists():
+            # Creates Ansible cache directory with right permissions
+            # if it doesn't exist yet.
+            display.vvv("Cache directory doesn't exist. Creating it.")
+            try:
+                self._uri.mkdir(mode=0o700, parents=True)
+            except Exception as e:
+                raise AnsibleError("""Unable to create {dir}.
+                Original error was : {err}""".format(dir=self._uri,
+                                                     err=to_native(e)))
+
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        config_manager = ConfigManager()
+        config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
+        self._config = ConfigParser()
+        self._config.read(config_file)
+
+        display.vvv("Using {} as configuration file.".format(config_file))
+
+        self._api_hostname = None
+        self._api_username = None
+        self._api_password = None
+        self._use_cpasswords = None
+        self._cache_plugin = None
+        self._cache = None
+        self._timeout = 86400  # 1 day
+        self._cachetoken = None
+        self._timeouttoken = TIME_FOR_RENEW  # 12 hours
+
+        if self._config.has_section("re2o"):
+            display.vvv("Found section re2o in configuration file")
+
+            self._api_hostname = self._readconfig(key="api_hostname")
+            self._use_cpasswords = self._readconfig(key="use_cpasswords",
+                                                    boolean=True)
+            self._cache_plugin = self._readconfig(key="cache")
+            self._timeout = self._readconfig(key="timeout", integer=True,
+                                             default=86400)
+            self._timeouttoken = self._readconfig(key="timeout_token",
+                                                  integer=True,
+                                                  default=TIME_FOR_RENEW)
+
+        if self._cache_plugin is not None:
+            display.vvv("Using {} as cache plugin".format(self._cache_plugin))
+            cachedir = Path.home() / ".cache/ansible/re2oapi"
+
+            if self._cache_plugin == 'jsonfile':
+                self._manage_cachedir(cachedir=cachedir, plugin='json')
+            elif self._cache_plugin == 'yaml':
+                self._manage_cachedir(cachedir=cachedir, plugin='yaml')
+            elif self._cache_plugin == 'pickle':
+                self._manage_cachedir(cachedir=cachedir, plugin='pickle')
+            elif self._cache_plugin == 'memcached':
+                # requires packages python3-memcache and memcached
+                display.vvvv("Please make sure you have installed packages"
+                             "python3-memcache and memcached"
+                             )
+                self._uri = self._readconfig(key='memcached_connection',
+                                                  default=['127.0.0.1:11211'],
+                                                  )
+            else:
+                raise AnsibleError("Cache plugin {} not supported"
+                                   .format(self._cache_plugin))
+
+            self._cache = cache_loader.get(self._cache_plugin,
+                                           _uri=self._uri,
+                                           _timeout=self._timeout,
+                                           )
+            self._cachetoken = cache_loader.get(self._cache_plugin,
+                                                _uri=self._uri,
+                                                _timeout=self._timeouttoken,
+                                                )
+
+
     def run(self, terms, variables=None, api_hostname=None, api_username=None,
             api_password=None, use_tls=True):
 
@@ -354,33 +475,20 @@ class LookupModule(LookupBase):
            :returns: A list of results to the specific queries.
         """
 
-        config_manager = ConfigManager()
+        # Use the hostname specified by the user if it exists.
-        config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
+        if api_hostname is not None:
-        config = ConfigParser()
+            display.vvv("Overriding api_hostname with {}".format(api_hostname))
-        config.read(config_file)
+        else:
+            api_hostname = self._api_hostname
 
-        use_cpasswords = False
+        if self._api_hostname is None:
-
-        if config.has_section("re2o"):
-            display.vvv("Found section re2o in configuration file")
-            if config.has_option("re2o", "api_hostname"):
-                display.vvv("Found option api_hostname in config file")
-                api_hostname = config.get("re2o", "api_hostname")
-                display.vvv("Override api_hostname with {} from configuration"
-                            .format(api_hostname))
-            if config.has_option("re2o", "use_cpasswords"):
-                display.vvv("Found option use_cpasswords in config file")
-                use_cpasswords = config.getboolean("re2o", "use_cpasswords")
-                display.vvv("Override api_hostname with {} from configuration"
-                            .format(use_cpasswords))
-
-        if api_hostname is None:
             raise AnsibleError(to_native(
                 'You must specify a hostname to contact re2oAPI'
             ))
 
-        if api_username is None and api_password is None and use_cpasswords:
+        if (api_username is None and api_password is None
-            display.vvv("Use cpasswords vault to get API credentials.")
+                and self._use_cpasswords):
+            display.vvv("Using cpasswords vault to get API credentials.")
             api_username = variables.get('vault_re2o_service_user')
             api_password = variables.get('vault_re2o_service_password')
 
@@ -394,12 +502,12 @@ class LookupModule(LookupBase):
                 'You must specify a valid password to connect to re2oAPI'
             ))
 
-        api_client = Client(api_hostname, api_username,
+        api_client = Client(api_hostname, api_username, api_password,
-                            api_password, use_tls=True)
+                            use_tls=True, cachetoken=self._cachetoken)
 
         res = []
         dterms = collections.deque(terms)
-        machines_roles = None  # TODO : Cache this.
+
         display.vvv("Lookup terms are {}".format(terms))
         while dterms:
             term = dterms.popleft()
@@ -411,10 +519,7 @@ class LookupModule(LookupBase):
             elif term == 'get_role':
                 try:
                     role_name = dterms.popleft()
-                    roles, machines_roles = self._get_role(api_client,
+                    roles = self._get_role(api_client, role_name)
-                                                           role_name,
-                                                           machines_roles,
-                                                           )
                     res.append(roles)
                 except IndexError:
                     display.v("Error in re2oapi : No role_name provided")
@@ -429,59 +534,157 @@ class LookupModule(LookupBase):
                                        .format(to_native(e)))
         return res
 
+    def _get_cache(self, key):
+        if self._cache:
+            return self._cache.get(key)
+        else:
+            return None
+
+    def _set_cache(self, key, value):
+        if self._cache:
+            return self._cache.set(key, value)
+        else:
+            return None
+
+    def _is_cached(self, key):
+        if self._cache:
+            return self._cache.contains(key)
+        else:
+            return False
+
     def _getzones(self, api_client):
         display.v("Getting dns zone names")
-        zones = api_client.list('dns/zones')
+        zones, zones_name = None, None
-        zones_name = [zone["name"][1:] for zone in zones]
+
+        if self._is_cached('dnszones'):
+            zones_name = self._get_cache('dnszones')
+
+        if zones_name is not None:
+            display.vvv("Found dnszones in cache.")
+
+        else:
+            if self._is_cached('dns_zones'):
+                zones = self._get_cache('dns_zones')
+            if zones is not None:
+                display.vvv("Found dns/zones in cache.")
+            else:
+                display.vvv("Contacting the API, endpoint dns/zones...")
+                zones = api_client.list('dns/zones')
+                display.vvv("...Done")
+            zones_name = [zone["name"][1:] for zone in zones]
+            display.vvv("Storing dnszones in cache.")
+            self._set_cache('dnszones', zones_name)
+        display.vvv('\n')
         return zones_name
 
     def _getreverse(self, api_client):
         display.v("Getting dns reverse zones")
-        display.vvv("Contacting the API, endpoint dns/reverse-zones...")
+
-        zones = api_client.list('dns/reverse-zones')
+        zones, res = None, None
-        display.vvv("...Done")
+
-        res = []
+        if self._is_cached('dnsreverse'):
-        for zone in zones:
+            res = self._get_cache('dnsreverse')
-            if zone['ptr_records']:
+
-                display.vvv('Found PTR records')
+        if res is not None:
-                subnets = []
+            display.vvv("Found dnsreverse in cache.")
-                for net in zone['cidrs']:
+
-                    net = netaddr.IPNetwork(net)
+        else:
-                    if net.prefixlen > 24:
+            if self._is_cached('dns_reverse-zones'):
-                        subnets.extend(net.subnet(32))
+                zones = self._get_cache('dns_reverse-zones')
-                    elif net.prefixlen > 16:
+
-                        subnets.extend(net.subnet(24))
+            if zones is not None:
-                    elif net.prefixlen > 8:
+                display.vvv("Found dns/reverse-zones in cache.")
-                        subnets.extend(net.subnet(16))
+            else:
-                    else:
+                display.vvv("Contacting the API, endpoint dns/reverse-zones..")
-                        subnets.extend(net.subnet(8))
+                zones = api_client.list('dns/reverse-zones')
-                for subnet in subnets:
+                display.vvv("...Done")
-                    _address = netaddr.IPAddress(subnet.first)
+
-                    rev_dns_a = _address.reverse_dns.split('.')[:-1]
+            display.vvv("Trying to format dns reverse in a nice way.")
-                    if subnet.prefixlen == 8:
+            res = []
-                        zone_name = '.'.join(rev_dns_a[3:])
+            for zone in zones:
-                    elif subnet.prefixlen == 16:
+                if zone['ptr_records']:
-                        zone_name = '.'.join(rev_dns_a[2:])
+                    display.vvv('Found PTR records')
-                    elif subnet.prefixlen == 24:
+                    subnets = []
-                        zone_name = '.'.join(rev_dns_a[1:])
+                    for net in zone['cidrs']:
-                    res.append(zone_name)
+                        net = netaddr.IPNetwork(net)
-                    display.vvv("Found reverse zone {}".format(zone_name))
+                        if net.prefixlen > 24:
+                            subnets.extend(net.subnet(32))
+                        elif net.prefixlen > 16:
+                            subnets.extend(net.subnet(24))
+                        elif net.prefixlen > 8:
+                            subnets.extend(net.subnet(16))
+                        else:
+                            subnets.extend(net.subnet(8))
+
+                    for subnet in subnets:
+                        _address = netaddr.IPAddress(subnet.first)
+                        rev_dns_a = _address.reverse_dns.split('.')[:-1]
+                        if subnet.prefixlen == 8:
+                            zone_name = '.'.join(rev_dns_a[3:])
+                        elif subnet.prefixlen == 16:
+                            zone_name = '.'.join(rev_dns_a[2:])
+                        elif subnet.prefixlen == 24:
+                            zone_name = '.'.join(rev_dns_a[1:])
+                        res.append(zone_name)
+                        display.vvv("Found reverse zone {}".format(zone_name))
+
                 if zone['ptr_v6_records']:
                     display.vvv("Found PTR v6 record")
-                    net = netaddr.IPNetwork(zone['prefix_v6']+'/'+str(zone['prefix_v6_length']))
+                    net = netaddr.IPNetwork(zone['prefix_v6']
-                    net_class = max(((net.prefixlen -1) // 4) +1, 1)
+                                            + '/'
+                                            + str(zone['prefix_v6_length']))
+                    net_class = max(((net.prefixlen - 1) // 4) + 1, 1)
                     zone6_name = ".".join(
-                        netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:])[:-1]
+                        netaddr.IPAddress(net.first)
+                        .reverse_dns.split('.')[32 - net_class:])[:-1]
                     res.append(zone6_name)
                     display.vvv("Found reverse zone {}".format(zone6_name))
-        return list(set(res))
+
+            display.vvv("Storing dns reverse zones in cache.")
+            self._set_cache('dnsreverse', list(set(res)))
+
+        display.vvv('\n')
+        return res
 
     def _rawquery(self, api_client, endpoint):
-        display.v("Make a raw query to endpoint {}".format(endpoint))
+        res = None
-        return api_client.list(endpoint)
+        if self._is_cached(endpoint.replace('/', '_')):
+            res = self._get_cache(endpoint.replace('/', '_'))
+        if res is not None:
+            display.vvv("Found {} in cache.".format(endpoint))
+        else:
+            display.v("Making a raw query {host}/api/{endpoint}"
+                      .format(host=self.api_hostname, endpoint=endpoint))
+            res = api_client.list(endpoint)
+            display.vvv("Storing result in cache.")
+            self._set_cache(endpoint.replace('/', '_'), res)
 
-    def _get_role(self, api_client, role_name, machines_roles):
+        display.vvv('\n')
-        if machines_roles is None:
+        return res
-            machines_roles = api_client.list("machines/role")
+
-        return list(filter(lambda machine: machine["role_type"] == role_name,
+    def _get_role(self, api_client, role_name):
-                           machines_roles)), machines_roles
+        res, machines_roles = None, None
+
+        if self._is_cached(role_name):
+            res = self._get_cache(role_name)
+
+        if res is not None:
+            display.vvv("Found {} in cache.".format(role_name))
+        else:
+            if self._is_cached("machines_role"):
+                machines_roles = self._get_cache("machines_role")
+
+            if machines_roles is not None:
+                display.vvv("Found machines/roles in cache.")
+            else:
+                machines_roles = api_client.list("machines/role")
+                display.vvv("Storing machines/role in cache.")
+                self._set_cache("machines_role", machines_roles)
+
+            res = list(filter(lambda m: m["role_type"] == role_name,
+                              machines_roles))
+            display.vvv("Storing {} in cache.".format(role_name))
+            self._set_cache(role_name, res)
+
+        display.vvv('\n')
+        return res