[re2o_lookup] Make use of cache.
parent
2d417ba516
commit
f4326afd76
|
|
@ -45,3 +45,10 @@ api_hostname = intranet.crans.org
|
|||
|
||||
# Whether or not using vault_cranspasswords
|
||||
use_cpasswords = True
|
||||
|
||||
# Specify cache plugin for re2o API. By default, cache nothing
|
||||
cache = jsonfile
|
||||
|
||||
# Time in second before the cache expired. 0 means never expire cache.
|
||||
# Default is 120 seconds.
|
||||
timeout = 120
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ For a detailed example look at https://github.com/ansible/ansible/blob/3dbf89e8a
|
|||
The API Client has been adapted from https://gitlab.federez.net/re2o/re2oapi
|
||||
"""
|
||||
|
||||
from ansible.plugins.loader import cache_loader
|
||||
|
||||
from pathlib import Path
|
||||
import datetime
|
||||
import requests
|
||||
|
|
@ -340,6 +342,73 @@ class LookupModule(LookupBase):
|
|||
- debug: var=dnszones
|
||||
"""
|
||||
|
||||
def _readconfig(self, section="re2o", key=None, boolean=False,
|
||||
integer=False):
|
||||
config = self._config
|
||||
if not config:
|
||||
return None
|
||||
else:
|
||||
if config.has_option(section, key):
|
||||
display.vvv("Found key {} in configuration file".format(key))
|
||||
if boolean:
|
||||
return config.getboolean(section, key)
|
||||
elif integer:
|
||||
return config.getint(section, key)
|
||||
else:
|
||||
return config.get(section, key)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
|
||||
self._config = ConfigParser()
|
||||
self._config.read(config_file)
|
||||
|
||||
display.vvv("Using {} as configuration file.".format(config_file))
|
||||
|
||||
self._api_hostname = None
|
||||
self._api_username = None
|
||||
self._api_password = None
|
||||
self._use_cpasswords = None
|
||||
self._cache_plugin = None
|
||||
self._cache = None
|
||||
self._timeout = 120
|
||||
|
||||
if self._config.has_section("re2o"):
|
||||
display.vvv("Found section re2o in configuration file")
|
||||
|
||||
self._api_hostname = self._readconfig(key="api_hostname")
|
||||
self._use_cpasswords = self._readconfig(key="use_cpasswords",
|
||||
boolean=True)
|
||||
self._cache_plugin = self._readconfig(key="cache")
|
||||
self._timeout = self._readconfig(key="timeout", integer=True)
|
||||
|
||||
if self._cache_plugin is not None:
|
||||
display.vvv("Using {} as cache plugin".format(self._cache_plugin))
|
||||
|
||||
if self._cache_plugin == 'jsonfile':
|
||||
self._cachedir = Path.home() / ".cache/Ansible/re2oapi"
|
||||
display.vvv("Cache directory is {}".format(self._cachedir))
|
||||
if not self._cachedir.exists():
|
||||
# Creates Ansible cache directory with right permissions
|
||||
# if it doesn't exist yet.
|
||||
display.vvv("Cache directory doesn't exist. Creating it.")
|
||||
try:
|
||||
self._cachedir.mkdir(mode=0o700, parents=True)
|
||||
except Exception as e:
|
||||
raise AnsibleError("""Unable to create {dir}.
|
||||
Original error was : {err}"""
|
||||
.format(dir=self._cachedir,
|
||||
err=to_native(e)))
|
||||
self._cache = cache_loader.get('jsonfile',
|
||||
_uri=self._cachedir,
|
||||
_timeout=self._timeout,
|
||||
)
|
||||
else:
|
||||
raise AnsibleError("Cache plugin {} not supported"
|
||||
.format(self._cache_plugin))
|
||||
|
||||
def run(self, terms, variables=None, api_hostname=None, api_username=None,
|
||||
api_password=None, use_tls=True):
|
||||
|
||||
|
|
@ -354,33 +423,20 @@ class LookupModule(LookupBase):
|
|||
:returns: A list of results to the specific queries.
|
||||
"""
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
|
||||
config = ConfigParser()
|
||||
config.read(config_file)
|
||||
# Use the hostname specified by the user if it exists.
|
||||
if api_hostname is not None:
|
||||
display.vvv("Overriding api_hostname with {}".format(api_hostname))
|
||||
else:
|
||||
api_hostname = self._api_hostname
|
||||
|
||||
use_cpasswords = False
|
||||
|
||||
if config.has_section("re2o"):
|
||||
display.vvv("Found section re2o in configuration file")
|
||||
if config.has_option("re2o", "api_hostname"):
|
||||
display.vvv("Found option api_hostname in config file")
|
||||
api_hostname = config.get("re2o", "api_hostname")
|
||||
display.vvv("Override api_hostname with {} from configuration"
|
||||
.format(api_hostname))
|
||||
if config.has_option("re2o", "use_cpasswords"):
|
||||
display.vvv("Found option use_cpasswords in config file")
|
||||
use_cpasswords = config.getboolean("re2o", "use_cpasswords")
|
||||
display.vvv("Override api_hostname with {} from configuration"
|
||||
.format(use_cpasswords))
|
||||
|
||||
if api_hostname is None:
|
||||
if self._api_hostname is None:
|
||||
raise AnsibleError(to_native(
|
||||
'You must specify a hostname to contact re2oAPI'
|
||||
))
|
||||
|
||||
if api_username is None and api_password is None and use_cpasswords:
|
||||
display.vvv("Use cpasswords vault to get API credentials.")
|
||||
if (api_username is None and api_password is None
|
||||
and self._use_cpasswords):
|
||||
display.vvv("Using cpasswords vault to get API credentials.")
|
||||
api_username = variables.get('vault_re2o_service_user')
|
||||
api_password = variables.get('vault_re2o_service_password')
|
||||
|
||||
|
|
@ -399,7 +455,7 @@ class LookupModule(LookupBase):
|
|||
|
||||
res = []
|
||||
dterms = collections.deque(terms)
|
||||
machines_roles = None # TODO : Cache this.
|
||||
|
||||
display.vvv("Lookup terms are {}".format(terms))
|
||||
while dterms:
|
||||
term = dterms.popleft()
|
||||
|
|
@ -411,10 +467,7 @@ class LookupModule(LookupBase):
|
|||
elif term == 'get_role':
|
||||
try:
|
||||
role_name = dterms.popleft()
|
||||
roles, machines_roles = self._get_role(api_client,
|
||||
role_name,
|
||||
machines_roles,
|
||||
)
|
||||
roles = self._get_role(api_client, role_name)
|
||||
res.append(roles)
|
||||
except IndexError:
|
||||
display.v("Error in re2oapi : No role_name provided")
|
||||
|
|
@ -429,17 +482,72 @@ class LookupModule(LookupBase):
|
|||
.format(to_native(e)))
|
||||
return res
|
||||
|
||||
def _get_cache(self, key):
|
||||
if self._cache:
|
||||
return self._cache.get(key)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _set_cache(self, key, value):
|
||||
if self._cache:
|
||||
return self._cache.set(key, value)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _is_cached(self, key):
|
||||
if self._cache:
|
||||
return self._cache.contains(key)
|
||||
else:
|
||||
return False
|
||||
|
||||
def _getzones(self, api_client):
|
||||
display.v("Getting dns zone names")
|
||||
zones, zones_name = None, None
|
||||
|
||||
if self._is_cached('dnszones'):
|
||||
zones_name = self._get_cache('dnszones')
|
||||
|
||||
if zones_name is not None:
|
||||
display.vvv("Found dnszones in cache.")
|
||||
|
||||
else:
|
||||
if self._is_cached('dns_zones'):
|
||||
zones = self._get_cache('dns_zones')
|
||||
if zones is not None:
|
||||
display.vvv("Found dns/zones in cache.")
|
||||
else:
|
||||
display.vvv("Contacting the API, endpoint dns/zones...")
|
||||
zones = api_client.list('dns/zones')
|
||||
display.vvv("...Done")
|
||||
zones_name = [zone["name"][1:] for zone in zones]
|
||||
display.vvv("Storing dnszones in cache.")
|
||||
self._set_cache('dnszones', zones_name)
|
||||
|
||||
return zones_name
|
||||
|
||||
def _getreverse(self, api_client):
|
||||
display.v("Getting dns reverse zones")
|
||||
display.vvv("Contacting the API, endpoint dns/reverse-zones...")
|
||||
|
||||
zones, res = None, None
|
||||
|
||||
if self._is_cached('dnsreverse'):
|
||||
res = self._get_cache('dnsreverse')
|
||||
|
||||
if res is not None:
|
||||
display.vvv("Found dnsreverse in cache.")
|
||||
|
||||
else:
|
||||
if self._is_cached('dns_reverse-zones'):
|
||||
zones = self._get_cache('dns_reverse-zones')
|
||||
|
||||
if zones is not None:
|
||||
display.vvv("Found dns/reverse-zones in cache.")
|
||||
else:
|
||||
display.vvv("Contacting the API, endpoint dns/reverse-zones..")
|
||||
zones = api_client.list('dns/reverse-zones')
|
||||
display.vvv("...Done")
|
||||
|
||||
display.vvv("Trying to format dns reverse in a nice way.")
|
||||
res = []
|
||||
for zone in zones:
|
||||
if zone['ptr_records']:
|
||||
|
|
@ -455,6 +563,7 @@ class LookupModule(LookupBase):
|
|||
subnets.extend(net.subnet(16))
|
||||
else:
|
||||
subnets.extend(net.subnet(8))
|
||||
|
||||
for subnet in subnets:
|
||||
_address = netaddr.IPAddress(subnet.first)
|
||||
rev_dns_a = _address.reverse_dns.split('.')[:-1]
|
||||
|
|
@ -466,22 +575,60 @@ class LookupModule(LookupBase):
|
|||
zone_name = '.'.join(rev_dns_a[1:])
|
||||
res.append(zone_name)
|
||||
display.vvv("Found reverse zone {}".format(zone_name))
|
||||
|
||||
if zone['ptr_v6_records']:
|
||||
display.vvv("Found PTR v6 record")
|
||||
net = netaddr.IPNetwork(zone['prefix_v6']+'/'+str(zone['prefix_v6_length']))
|
||||
net = netaddr.IPNetwork(zone['prefix_v6']
|
||||
+ '/'
|
||||
+ str(zone['prefix_v6_length']))
|
||||
net_class = max(((net.prefixlen - 1) // 4) + 1, 1)
|
||||
zone6_name = ".".join(
|
||||
netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:])[:-1]
|
||||
netaddr.IPAddress(net.first)
|
||||
.reverse_dns.split('.')[32 - net_class:])[:-1]
|
||||
res.append(zone6_name)
|
||||
display.vvv("Found reverse zone {}".format(zone6_name))
|
||||
return list(set(res))
|
||||
|
||||
display.vvv("Storing dns reverse zones in cache.")
|
||||
self._set_cache('dnsreverse', list(set(res)))
|
||||
|
||||
return res
|
||||
|
||||
def _rawquery(self, api_client, endpoint):
|
||||
display.v("Make a raw query to endpoint {}".format(endpoint))
|
||||
return api_client.list(endpoint)
|
||||
res = None
|
||||
if self._is_cached(endpoint.replace('/', '_')):
|
||||
res = self._get_cache(endpoint.replace('/', '_'))
|
||||
if res is not None:
|
||||
display.vvv("Found {} in cache.".format(endpoint))
|
||||
else:
|
||||
display.v("Making a raw query {host}/api/{endpoint}"
|
||||
.format(host=self.api_hostname, endpoint=endpoint))
|
||||
res = api_client.list(endpoint)
|
||||
display.vvv("Storing result in cache.")
|
||||
self._set_cache(endpoint.replace('/', '_'), res)
|
||||
return res
|
||||
|
||||
def _get_role(self, api_client, role_name, machines_roles):
|
||||
if machines_roles is None:
|
||||
def _get_role(self, api_client, role_name):
|
||||
res, machines_roles = None, None
|
||||
|
||||
if self._is_cached(role_name):
|
||||
res = self._get_cache(role_name)
|
||||
|
||||
if res is not None:
|
||||
display.vvv("Found {} in cache.".format(role_name))
|
||||
else:
|
||||
if self._is_cached("machines_role"):
|
||||
machines_roles = self._get_cache("machines_role")
|
||||
|
||||
if machines_roles is not None:
|
||||
display.vvv("Found machines/roles in cache.")
|
||||
else:
|
||||
machines_roles = api_client.list("machines/role")
|
||||
return list(filter(lambda machine: machine["role_type"] == role_name,
|
||||
machines_roles)), machines_roles
|
||||
display.vvv("Storing machines/role in cache.")
|
||||
self._set_cache("machines_role", machines_roles)
|
||||
|
||||
res = list(filter(lambda m: m["role_type"] == role_name,
|
||||
machines_roles))
|
||||
display.vvv("Storing {} in cache.".format(role_name))
|
||||
self._set_cache(role_name, res)
|
||||
|
||||
return res
|
||||
|
|
|
|||
Loading…
Reference in New Issue