crans
/
ansible
mirror of http://gitlab.adm.crans.org/nounous/ansible.git
19
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'cache_lookup_api' into 'master' 

Cache lookup api

See merge request nounous/ansible!18
certbot_on_virtu
_pollion 2020-05-10 12:56:00 +02:00
parent 90d25818e1 c042303665
commit e55d74bb66
2 changed files with 326 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ansible.cfg
View File

@ -45,3 +45,18 @@ api_hostname = intranet.crans.org
# Whether or not using vault_cranspasswords
use_cpasswords = True
# Specify cache plugin for re2o API. By default, cache nothing
cache = jsonfile
# Only used for memcached plugin
# List of connection information for the memcached DBs
# Default is ['127.0.0.1:11211']
# memcached_connection = ['127.0.0.1:11211']
# Time in second before the cache expired. 0 means never expire cache.
# Default is 24 hours.
timeout = 86400
# Default is 12 hours.
timeout_token = 43200

419
lookup_plugins/re2oapi.py
View File

@ -7,6 +7,8 @@ For a detailed example look at https://github.com/ansible/ansible/blob/3dbf89e8a
The API Client has been adapted from https://gitlab.federez.net/re2o/re2oapi
"""
from ansible.plugins.loader import cache_loader
from pathlib import Path
import datetime
import requests
@ -28,38 +30,67 @@ from ansible.config.manager import ConfigManager
# Ansible Logger to stdout
display = Display()
# Number of seconds before expiration where renewing the token is done
TIME_FOR_RENEW = 120
# Default name of the file to store tokens. Path $HOME/{DEFAUlt_TOKEN_FILENAME}
DEFAULT_TOKEN_FILENAME = '.re2o.token'
# If no plugin is used, then use this as token timeout.
# Overriden by key timeout_token from ansible configuration.
TIME_FOR_RENEW = 43200 # 12 jours
class Client:
"""
Class based client to contact re2o API.
"""
def __init__(self, hostname, username, password, use_tls=True):
def __init__(self, hostname, username, password,
use_tls=True, cachetoken=None):
"""
:arg hostname: The hostname of the Re2o instance to use.
:arg username: The username to use.
:arg password: The password to use.
:arg use_tls: A boolean to specify whether the client should use a
a TLS connection. Default is True. Please, keep it.
:arg cachetoken: The cache to use to manage authentication token.
If it is None, then store the token in a file.
"""
self.use_tls = use_tls
self.hostname = hostname
self._username = username
self._password = password
self.token_file = Path.home() / DEFAULT_TOKEN_FILENAME
self._cachetoken = cachetoken
self.token_file = None
if self._cachetoken is None:
self.token_file = Path.home() / DEFAULT_TOKEN_FILENAME
display.vvv("Setting token file to {}".format(self.token_file))
else:
try:
display.vvv("Using {} as cache plugin"
.format(self._cachetoken.plugin_name))
except AttributeError:
# Happens when plugin_name is not implemented...
# For example with memcached
display.vvv("Using cache plugin specified in configuration.")
display.v("Connecting to {hostname} as user {user}".format(
hostname=to_native(self.hostname), user=to_native(self._username)))
try:
self.token = self._get_token_from_file()
except AnsibleFileNotFound:
display.vv("Force renew the token")
self._force_renew_token()
@property
def token(self):
if self._cachetoken:
display.vvv("Trying to get token from cache.")
if self._cachetoken.contains("auth_token"):
display.vvv("Found token in cache.")
return self._cachetoken.get("auth_token")
else:
display.vvv("Token not found. Forcing renew.")
return self._force_renew_token()
else:
try:
token = self._get_token_from_file()
if token['expiration'] < datetime.datetime.now() + \
datetime.timedelta(seconds=TIME_FOR_RENEW):
return self._force_renew_token()
except AnsibleError:
return self._force_renew_token()
def _get_token_from_file(self):
display.vv("Trying to fetch token from {}".format(self.token_file))
@ -93,13 +124,18 @@ class Client:
)
)
else:
display.vv("""Token successfully retreived from
file {token}""".format(token=self.token_file))
display.vv("Token successfully retreived from "
"file {token}".format(token=self.token_file))
return ret
def _force_renew_token(self):
self.token = self._get_token_from_server()
self._save_token_to_file()
token = self._get_token_from_server()
if self._cachetoken:
display.vvv("Storing authentication token in cache")
self._cachetoken.set("auth_token", token.get('token'))
else:
self._save_token_to_file(token)
return token.get('token')
def _get_token_from_server(self):
display.vv("Requesting a new token for {user}@{host}".format(
@ -139,7 +175,7 @@ class Client:
def _parse_date(self, date, date_format="%Y-%m-%dT%H:%M:%S"):
return datetime.datetime.strptime(date.split('.')[0], date_format)
def _save_token_to_file(self):
def _save_token_to_file(self, token):
display.vv("Saving token to file {}".format(self.token_file))
try:
# Read previous data to avoid erasures
@ -153,8 +189,8 @@ class Client:
if self.hostname not in data.keys():
data[self.hostname] = {}
data[self.hostname][self._username] = {
'token': self.token['token'],
'expiration': self.token['expiration'].isoformat(),
'token': token['token'],
'expiration': token['expiration'].isoformat(),
}
try:
@ -169,22 +205,6 @@ class Client:
display.vv("Token successfully written to file {}"
.format(self.token_file))
def get_token(self):
"""
Retrieves the token to use for the current connection.
Automatically renewed if needed.
"""
if self.need_renew_token:
self._force_renew_token()
return self.token['token']
@property
def need_renew_token(self):
return self.token['expiration'] < \
datetime.datetime.now() + \
datetime.timedelta(seconds=TIME_FOR_RENEW)
def _request(self, method, url, headers={}, params={}, *args, **kwargs):
display.vv("Building the {method} request to {url}.".format(
method=method.upper(),
@ -192,9 +212,9 @@ class Client:
))
# Force the 'Authorization' field with the right token.
display.vvv("Forcing authentication token.")
display.vvv("Forcing authentication token in headers.")
headers.update({
'Authorization': 'Token {}'.format(self.get_token())
'Authorization': 'Token {}'.format(self.token)
})
# Use a json format unless the user already specified something
@ -213,10 +233,10 @@ class Client:
# Force re-login to the server (case of a wrong token but valid
# credentials) and then retry the request without catching errors.
display.vv("Token refused. Trying to refresh the token.")
self._force_renew_token()
token = self._force_renew_token()
headers.update({
'Authorization': 'Token {}'.format(self.get_token())
'Authorization': 'Token {}'.format(token)
})
display.vv("Re-performing the request {method} {url}".format(
method=method.upper(),
@ -340,6 +360,107 @@ class LookupModule(LookupBase):
- debug: var=dnszones
"""
def _readconfig(self, section="re2o", key=None, default=None,
boolean=False, integer=False):
config = self._config
if not config:
return default
else:
if config.has_option(section, key):
display.vvv("Found key {} in configuration file".format(key))
if boolean:
return config.getboolean(section, key)
elif integer:
return config.getint(section, key)
else:
return config.get(section, key)
else:
return default
def _manage_cachedir(self, cachedir=None, plugin=None):
try:
self._uri = cachedir / plugin
except Exception:
raise AnsibleError("Undefined specification for cache plugin")
display.vvv("Cache directory is {}".format(self._uri))
if not self._uri.exists():
# Creates Ansible cache directory with right permissions
# if it doesn't exist yet.
display.vvv("Cache directory doesn't exist. Creating it.")
try:
self._uri.mkdir(mode=0o700, parents=True)
except Exception as e:
raise AnsibleError("""Unable to create {dir}.
Original error was : {err}""".format(dir=self._uri,
err=to_native(e)))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
config_manager = ConfigManager()
config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
self._config = ConfigParser()
self._config.read(config_file)
display.vvv("Using {} as configuration file.".format(config_file))
self._api_hostname = None
self._api_username = None
self._api_password = None
self._use_cpasswords = None
self._cache_plugin = None
self._cache = None
self._timeout = 86400 # 1 day
self._cachetoken = None
self._timeouttoken = TIME_FOR_RENEW # 12 hours
if self._config.has_section("re2o"):
display.vvv("Found section re2o in configuration file")
self._api_hostname = self._readconfig(key="api_hostname")
self._use_cpasswords = self._readconfig(key="use_cpasswords",
boolean=True)
self._cache_plugin = self._readconfig(key="cache")
self._timeout = self._readconfig(key="timeout", integer=True,
default=86400)
self._timeouttoken = self._readconfig(key="timeout_token",
integer=True,
default=TIME_FOR_RENEW)
if self._cache_plugin is not None:
display.vvv("Using {} as cache plugin".format(self._cache_plugin))
cachedir = Path.home() / ".cache/ansible/re2oapi"
if self._cache_plugin == 'jsonfile':
self._manage_cachedir(cachedir=cachedir, plugin='json')
elif self._cache_plugin == 'yaml':
self._manage_cachedir(cachedir=cachedir, plugin='yaml')
elif self._cache_plugin == 'pickle':
self._manage_cachedir(cachedir=cachedir, plugin='pickle')
elif self._cache_plugin == 'memcached':
# requires packages python3-memcache and memcached
display.vvvv("Please make sure you have installed packages"
"python3-memcache and memcached"
)
self._uri = self._readconfig(key='memcached_connection',
default=['127.0.0.1:11211'],
)
else:
raise AnsibleError("Cache plugin {} not supported"
.format(self._cache_plugin))
self._cache = cache_loader.get(self._cache_plugin,
_uri=self._uri,
_timeout=self._timeout,
)
self._cachetoken = cache_loader.get(self._cache_plugin,
_uri=self._uri,
_timeout=self._timeouttoken,
)
def run(self, terms, variables=None, api_hostname=None, api_username=None,
api_password=None, use_tls=True):
@ -354,33 +475,20 @@ class LookupModule(LookupBase):
:returns: A list of results to the specific queries.
"""
config_manager = ConfigManager()
config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
config = ConfigParser()
config.read(config_file)
# Use the hostname specified by the user if it exists.
if api_hostname is not None:
display.vvv("Overriding api_hostname with {}".format(api_hostname))
else:
api_hostname = self._api_hostname
use_cpasswords = False
if config.has_section("re2o"):
display.vvv("Found section re2o in configuration file")
if config.has_option("re2o", "api_hostname"):
display.vvv("Found option api_hostname in config file")
api_hostname = config.get("re2o", "api_hostname")
display.vvv("Override api_hostname with {} from configuration"
.format(api_hostname))
if config.has_option("re2o", "use_cpasswords"):
display.vvv("Found option use_cpasswords in config file")
use_cpasswords = config.getboolean("re2o", "use_cpasswords")
display.vvv("Override api_hostname with {} from configuration"
.format(use_cpasswords))
if api_hostname is None:
if self._api_hostname is None:
raise AnsibleError(to_native(
'You must specify a hostname to contact re2oAPI'
))
if api_username is None and api_password is None and use_cpasswords:
display.vvv("Use cpasswords vault to get API credentials.")
if (api_username is None and api_password is None
and self._use_cpasswords):
display.vvv("Using cpasswords vault to get API credentials.")
api_username = variables.get('vault_re2o_service_user')
api_password = variables.get('vault_re2o_service_password')
@ -394,12 +502,12 @@ class LookupModule(LookupBase):
'You must specify a valid password to connect to re2oAPI'
))
api_client = Client(api_hostname, api_username,
api_password, use_tls=True)
api_client = Client(api_hostname, api_username, api_password,
use_tls=True, cachetoken=self._cachetoken)
res = []
dterms = collections.deque(terms)
machines_roles = None # TODO : Cache this.
display.vvv("Lookup terms are {}".format(terms))
while dterms:
term = dterms.popleft()
@ -411,10 +519,7 @@ class LookupModule(LookupBase):
elif term == 'get_role':
try:
role_name = dterms.popleft()
roles, machines_roles = self._get_role(api_client,
role_name,
machines_roles,
)
roles = self._get_role(api_client, role_name)
res.append(roles)
except IndexError:
display.v("Error in re2oapi : No role_name provided")
@ -429,59 +534,157 @@ class LookupModule(LookupBase):
.format(to_native(e)))
return res
def _get_cache(self, key):
if self._cache:
return self._cache.get(key)
else:
return None
def _set_cache(self, key, value):
if self._cache:
return self._cache.set(key, value)
else:
return None
def _is_cached(self, key):
if self._cache:
return self._cache.contains(key)
else:
return False
def _getzones(self, api_client):
display.v("Getting dns zone names")
zones = api_client.list('dns/zones')
zones_name = [zone["name"][1:] for zone in zones]
zones, zones_name = None, None
if self._is_cached('dnszones'):
zones_name = self._get_cache('dnszones')
if zones_name is not None:
display.vvv("Found dnszones in cache.")
else:
if self._is_cached('dns_zones'):
zones = self._get_cache('dns_zones')
if zones is not None:
display.vvv("Found dns/zones in cache.")
else:
display.vvv("Contacting the API, endpoint dns/zones...")
zones = api_client.list('dns/zones')
display.vvv("...Done")
zones_name = [zone["name"][1:] for zone in zones]
display.vvv("Storing dnszones in cache.")
self._set_cache('dnszones', zones_name)
display.vvv('\n')
return zones_name
def _getreverse(self, api_client):
display.v("Getting dns reverse zones")
display.vvv("Contacting the API, endpoint dns/reverse-zones...")
zones = api_client.list('dns/reverse-zones')
display.vvv("...Done")
res = []
for zone in zones:
if zone['ptr_records']:
display.vvv('Found PTR records')
subnets = []
for net in zone['cidrs']:
net = netaddr.IPNetwork(net)
if net.prefixlen > 24:
subnets.extend(net.subnet(32))
elif net.prefixlen > 16:
subnets.extend(net.subnet(24))
elif net.prefixlen > 8:
subnets.extend(net.subnet(16))
else:
subnets.extend(net.subnet(8))
for subnet in subnets:
_address = netaddr.IPAddress(subnet.first)
rev_dns_a = _address.reverse_dns.split('.')[:-1]
if subnet.prefixlen == 8:
zone_name = '.'.join(rev_dns_a[3:])
elif subnet.prefixlen == 16:
zone_name = '.'.join(rev_dns_a[2:])
elif subnet.prefixlen == 24:
zone_name = '.'.join(rev_dns_a[1:])
res.append(zone_name)
display.vvv("Found reverse zone {}".format(zone_name))
zones, res = None, None
if self._is_cached('dnsreverse'):
res = self._get_cache('dnsreverse')
if res is not None:
display.vvv("Found dnsreverse in cache.")
else:
if self._is_cached('dns_reverse-zones'):
zones = self._get_cache('dns_reverse-zones')
if zones is not None:
display.vvv("Found dns/reverse-zones in cache.")
else:
display.vvv("Contacting the API, endpoint dns/reverse-zones..")
zones = api_client.list('dns/reverse-zones')
display.vvv("...Done")
display.vvv("Trying to format dns reverse in a nice way.")
res = []
for zone in zones:
if zone['ptr_records']:
display.vvv('Found PTR records')
subnets = []
for net in zone['cidrs']:
net = netaddr.IPNetwork(net)
if net.prefixlen > 24:
subnets.extend(net.subnet(32))
elif net.prefixlen > 16:
subnets.extend(net.subnet(24))
elif net.prefixlen > 8:
subnets.extend(net.subnet(16))
else:
subnets.extend(net.subnet(8))
for subnet in subnets:
_address = netaddr.IPAddress(subnet.first)
rev_dns_a = _address.reverse_dns.split('.')[:-1]
if subnet.prefixlen == 8:
zone_name = '.'.join(rev_dns_a[3:])
elif subnet.prefixlen == 16:
zone_name = '.'.join(rev_dns_a[2:])
elif subnet.prefixlen == 24:
zone_name = '.'.join(rev_dns_a[1:])
res.append(zone_name)
display.vvv("Found reverse zone {}".format(zone_name))
if zone['ptr_v6_records']:
display.vvv("Found PTR v6 record")
net = netaddr.IPNetwork(zone['prefix_v6']+'/'+str(zone['prefix_v6_length']))
net_class = max(((net.prefixlen -1) // 4) +1, 1)
net = netaddr.IPNetwork(zone['prefix_v6']
+ '/'
+ str(zone['prefix_v6_length']))
net_class = max(((net.prefixlen - 1) // 4) + 1, 1)
zone6_name = ".".join(
netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:])[:-1]
netaddr.IPAddress(net.first)
.reverse_dns.split('.')[32 - net_class:])[:-1]
res.append(zone6_name)
display.vvv("Found reverse zone {}".format(zone6_name))
return list(set(res))
display.vvv("Storing dns reverse zones in cache.")
self._set_cache('dnsreverse', list(set(res)))
display.vvv('\n')
return res
def _rawquery(self, api_client, endpoint):
display.v("Make a raw query to endpoint {}".format(endpoint))
return api_client.list(endpoint)
res = None
if self._is_cached(endpoint.replace('/', '_')):
res = self._get_cache(endpoint.replace('/', '_'))
if res is not None:
display.vvv("Found {} in cache.".format(endpoint))
else:
display.v("Making a raw query {host}/api/{endpoint}"
.format(host=self.api_hostname, endpoint=endpoint))
res = api_client.list(endpoint)
display.vvv("Storing result in cache.")
self._set_cache(endpoint.replace('/', '_'), res)
def _get_role(self, api_client, role_name, machines_roles):
if machines_roles is None:
machines_roles = api_client.list("machines/role")
return list(filter(lambda machine: machine["role_type"] == role_name,
machines_roles)), machines_roles
display.vvv('\n')
return res
def _get_role(self, api_client, role_name):
res, machines_roles = None, None
if self._is_cached(role_name):
res = self._get_cache(role_name)
if res is not None:
display.vvv("Found {} in cache.".format(role_name))
else:
if self._is_cached("machines_role"):
machines_roles = self._get_cache("machines_role")
if machines_roles is not None:
display.vvv("Found machines/roles in cache.")
else:
machines_roles = api_client.list("machines/role")
display.vvv("Storing machines/role in cache.")
self._set_cache("machines_role", machines_roles)
res = list(filter(lambda m: m["role_type"] == role_name,
machines_roles))
display.vvv("Storing {} in cache.".format(role_name))
self._set_cache(role_name, res)
display.vvv('\n')
return res