From e7888a6b9c1b40728fc4358cd790cdb02aec54f4 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Wed, 15 Apr 2020 10:03:46 +0200 Subject: [PATCH] Add mtail role for thot --- monitoring.yml | 5 + roles/mtail/handlers/main.yml | 5 + roles/mtail/tasks/main.yml | 28 ++++ roles/mtail/templates/default/mtail.j2 | 28 ++++ roles/mtail/templates/mtail/dhcpd.mtail.j2 | 143 ++++++++++++++++++ .../templates/update-motd.d/05-service.j2 | 3 + 6 files changed, 212 insertions(+) create mode 100644 roles/mtail/handlers/main.yml create mode 100644 roles/mtail/tasks/main.yml create mode 100644 roles/mtail/templates/default/mtail.j2 create mode 100644 roles/mtail/templates/mtail/dhcpd.mtail.j2 create mode 100755 roles/mtail/templates/update-motd.d/05-service.j2 diff --git a/monitoring.yml b/monitoring.yml index e60b4f95..f2084bda 100755 --- a/monitoring.yml +++ b/monitoring.yml @@ -63,3 +63,8 @@ - hosts: redisdead.adm.crans.org roles: - prometheus-node-exporter-postfix + +# Monitor logs with mtail +- hosts: thot.adm.crans.org + roles: + - mtail diff --git a/roles/mtail/handlers/main.yml b/roles/mtail/handlers/main.yml new file mode 100644 index 00000000..ad7558b6 --- /dev/null +++ b/roles/mtail/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart mtail + service: + name: mtail + state: restarted diff --git a/roles/mtail/tasks/main.yml b/roles/mtail/tasks/main.yml new file mode 100644 index 00000000..b4a419bb --- /dev/null +++ b/roles/mtail/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Install mtail + apt: + update_cache: true + name: mtail + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Configure mtail + template: + src: default/mtail.j2 + dest: /etc/default/mtail + notify: Restart mtail + +- name: Copy mtail configurations + template: + src: "mtail/{{ item }}.j2" + dest: "/etc/mtail/{{ item }}" + loop: + - dhcpd.mtail + notify: Restart mtail + +- name: Indicate role in motd + template: + src: update-motd.d/05-service.j2 + dest: /etc/update-motd.d/05-mtail + mode: 0755 diff --git a/roles/mtail/templates/default/mtail.j2 b/roles/mtail/templates/default/mtail.j2 new file mode 100644 index 00000000..bb782a3f --- /dev/null +++ b/roles/mtail/templates/default/mtail.j2 @@ -0,0 +1,28 @@ +# {{ ansible_managed }} +# Set to 1 to start Mtail at boot. +ENABLED=1 + +## Log processing. + +# List of files to monitor (mandatory). +LOGS=/var/log/cablage/global.log + +## Metrics exporting. + +# HTTP port to listen on. (default "3903") +#PORT=3903 + +# Path to collectd unixsock to write metrics to. +#COLLECTD_SOCKETPATH= + +# Host:port to graphite carbon server to write metrics to. +#GRAPHITE_HOSTPORT= + +# Host:port to statsd server to write metrics to. +#STATSD_HOSTPORT= + +# Interval between metric pushes, in seconds (default 60) +#METRIC_PUSH_INTERVAL= + +# Extra command-line arguments to pass to the server. +EXTRA_ARGS="" diff --git a/roles/mtail/templates/mtail/dhcpd.mtail.j2 b/roles/mtail/templates/mtail/dhcpd.mtail.j2 new file mode 100644 index 00000000..c612a404 --- /dev/null +++ b/roles/mtail/templates/mtail/dhcpd.mtail.j2 @@ -0,0 +1,143 @@ +# {{ ansible_managed }} +# Copyright 2011 Google Inc. All Rights Reserved. +# This file is available under the Apache license. + +# Define the exported metric names. The `by' keyword indicates the metric has +# dimensions. For example, `request_total' counts the frequency of each +# request's "command". The name `command' will be exported as the label name +# for the metric. The command provided in the code below will be exported as +# the label value. +counter dhcpd_request_total by command +counter dhcpd_config_file_errors +counter dhcpd_peer_disconnects +counter dhcpd_dhcpdiscovers +counter dhcpd_bind_xid_mismatch +counter dhcpd_duplicate_lease +counter dhcpd_bad_udp_checksum +counter dhcpd_unknown_subnet +counter dhcpd_dhcpdiscover_nofree by network +counter dhcpd_unknown_lease by ip +counter dhcpd_update_rejected +counter dhcpd_failover_peer_timeout +counter dhcpd_ip_already_in_use +counter dhcpd_ip_abandoned by reason +counter dhcpd_invalid_state_transition +counter dhcpd_negative_poolreq by pool +counter dhcpd_lease_conflicts + +# The `syslog' decorator defines a procedure. When a block of mtail code is +# "decorated", it is called before entering the block. The block is entered +# when the keyword `next' is reached. +def syslog { + /^(?P(?P\w+\s+\d+\s+\d+:\d+:\d+)|(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[+-]\d{2}:\d{2}))/ + + /\s+(?:\w+@)?(?P[\w\.-]+)\s+(?P[\w\.-]+)(?:\[(?P\d+)\])?:\s+(?P.*)/ { + # If the legacy_date regexp matched, try this format. + len($legacy_date) > 0 { + strptime($2, "Jan _2 15:04:05") + } + # If the RFC3339 style matched, parse it this way. + len($rfc3339_date) > 0 { + strptime($rfc3339_date, "2006-01-02T15:04:05.999999999Z07:00") + } + # Call into the decorated block + next + } +} + +# Define some pattern constants for reuse in the patterns below. +const IP /\d+(\.\d+){3}/ +const MATCH_IP /(?P/ + IP + /)/ +const MATCH_NETWORK /(?P\d+(\.\d+){1,3}\/\d+)/ +const MATCH_MAC /(?P([\da-f]{2}:){5}[\da-f]{2})/ + +@syslog { + # Request + /(balanced|balancing|BOOTREPLY|BOOTREQUEST|DHCPACK|DHCPDECLINE|DHCPDISCOVER|DHCPINFORM|DHCPNAK|DHCPOFFER|DHCPRELEASE|DHCPREQUEST)/ { + # The lowercased name of the command matched in the regex is used to + # count the frequency of each command. An external collector can use + # this to compute the rate of each command independently. + dhcpd_request_total[tolower($1)]++ + + # DHCP Discover + /DHCPDISCOVER from / + MATCH_MAC { + # Counts the discovery requests. + dhcpd_dhcpdiscovers++ + + /network / + MATCH_NETWORK + /: no free leases/ { + # If the range is full, your clients may be having a bad time. + dhcpd_dhcpdiscover_nofree[$network]++ + } + } + + } + + # Config file errors + /Configuration file errors encountered -- exiting/ { + # Counting config parse errors can he useful for detecting bad config + # pushes that made it to production. + dhcpd_config_file_errors++ + } + + # Peer disconnects + /peer ([^:]+): disconnected/ { + dhcpd_peer_disconnects++ + } + + # XID mismatches + /bind update on / + IP + / got ack from (?P\w+): xid mismatch./ { + dhcpd_bind_xid_mismatch++ + } + + # Duplicate lease + /uid lease / + MATCH_IP + / for client / + MATCH_MAC + / is duplicate on / + MATCH_NETWORK { + dhcpd_duplicate_lease++ + } + + # Bad UDP Checksum + /(?P\d+) bad udp checksums in \d+ packets/ { + dhcpd_bad_udp_checksum += $count + } + + # Unknown subnet + /DHCPDISCOVER from / + MATCH_MAC + / via / + IP + /: unknown network segment/ { + dhcpd_unknown_subnet++ + } + + # Unknown lease + /DHCPREQUEST for / + IP + /\(/ + IP + /\) from / + MATCH_MAC + / via / + IP + /: unknown lease / + MATCH_IP { + dhcpd_unknown_lease[$ip]++ + } + + # Update rejected + /bind update on \S+ from \S+ rejected: incoming update is less critical than the outgoing update/ { + dhcpd_update_rejected++ + } + + /timeout waiting for failover peer \S+/ { + dhcpd_failover_peer_timeout++ + } + + /ICMP Echo reply while lease / + IP + /valid/ { + dhcpd_ip_already_in_use++ + } + + /unexpected ICMP Echo reply from / + IP { + dhcpd_ip_already_in_use++ + } + + /Abandoning IP address / + IP + /: (?P.*)/ { + dhcpd_ip_abandoned[$reason]++ + } + + /bind update on \S+ from \S+ rejected: / + IP + /: invalid state transition/ { + dhcpd_invalid_state_transition++ + } + + /peer (?P[^:]+): Got POOLREQ, answering negatively!/ { + dhcpd_negative_poolreq[$pool]++ + } + + /Lease conflict at/ { + dhcpd_lease_conflicts++ + } +} diff --git a/roles/mtail/templates/update-motd.d/05-service.j2 b/roles/mtail/templates/update-motd.d/05-service.j2 new file mode 100755 index 00000000..74dc66f5 --- /dev/null +++ b/roles/mtail/templates/update-motd.d/05-service.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/tail +3 +# {{ ansible_managed }} +> mtail a été déployé sur cette machine. Voir /etc/mtail/.