[mirror_backend]

group_vars/mirror_backend.yml

@@ -0,0 +1,90 @@
+glob_ftpsync:
+  root: /mirror/pub
+  mirror:
+    name: CRANS
+    info:
+      maintainer: Les Nounous <contact@crans.org>
+      country: FR
+      location: Cachan, Île-de-France
+  targets:
+    - name: main
+      dest: debian
+      cron_time: "25 1,13"
+      rsync_host: syncproxy.eu.debian.org
+      rsync_path: debian
+    - name: security
+      dest: debian-security
+      cron_time: "40    *"
+      rsync_host: rsync.security.debian.org
+      rsync_path: debian-security
+    - name: backports
+      dest: debian-backports
+      cron_time: " 7 3,15"
+      rsync_host: syncproxy.eu.debian.org
+      rsync_path: debian-backports
+
+glob_rsync_mirror:
+  root: /mirror/pub
+  targets:
+    - name: videolan
+      dest: videolan
+      cron_time: "03 10,14,18,22,2,6"
+      rsync_host: rsync.videolan.org
+      rsync_path: videolan-ftp
+    - name: debian
+      dest: distributions/linux/debian
+      cron_time: "00 5"
+      rsync_host: cdimage.debian.org
+      rsync_path: cdimage/release
+    - name: debian-cloud
+      dest: distributions/linux/debian/cloud
+      cron_time: "00 5"
+      rsync_host: cdimage.debian.org
+      rsync_path: cdimage/cloud/Openstack
+      exclude:
+        - archive
+    - name: ubuntu
+      dest: distributions/linux/ubuntu
+      cron_time: "00 5"
+      rsync_host: cdimage.ubuntu.com
+      rsync_path: cdimage/releases
+    - name: xubuntu
+      dest: distributions/linux/xubuntu
+      cron_time: "00 5"
+      rsync_host: cdimage.ubuntu.com
+      rsync_path: cdimage/xubuntu/releases
+    - name: kubuntu
+      dest: distributions/linux/kubuntu
+      cron_time: "00 5"
+      rsync_host: cdimage.ubuntu.com
+      rsync_path: cdimage/kubuntu/releases
+    - name: lubuntu
+      dest: distributions/linux/lubuntu
+      cron_time: "00 5"
+      rsync_host: cdimage.ubuntu.com
+      rsync_path: cdimage/lubuntu/releases
+    - name: ubuntu-mate
+      dest: distributions/linux/ubuntu-mate
+      cron_time: "00 5"
+      rsync_host: cdimage.ubuntu.com
+      rsync_path: cdimage/ubuntu-mate/releases
+
+glob_apt_mirror:
+  root: /mirror/pub
+  targets:
+    - name: grafana
+      symlink: ""
+      scheme: https
+      host: packages.grafana.com
+      path: oss/deb
+      suite: stable
+      components:
+        - main
+    - name: proxmox
+      symlink: ""
+      scheme: http
+      host: download.proxmox.com
+      path: debian/pve
+      suite: buster
+      components:
+        - pve-no-subscription

host_vars/charybde.adm.crans.org.yml

@@ -1,63 +1,47 @@
 ---
+loc_ftpsync:
+  root: /pool/mirror/pub
 
-to_backup:
+loc_rsync_mirror:
-  - {
+  root: /pool/mirror/pub
-  name: "var",
+
-  path: "/var",
+loc_apt_mirror:
-  auth_users: "backupcrans",
+  root: /pool/mirror/pub
-  secrets_file: "/etc/rsyncd.secrets",
-  hosts_allow: ["zephir.adm.crans.org", "10.231.136.6"],
-  }
-  - {
-  name: "slash",
-  path: "/",
-  auth_users: "backupcrans",
-  secrets_file: "/etc/rsyncd.secrets",
-  hosts_allow: ["zephir.adm.crans.org", "10.231.136.6"],
-  }
-  - {
-  name: "ftp",
-  path: "/pubftp",
-  comment: "CRANS FTP",
-  uid: "nobody",
-  gid: "nogroup",
-  hosts_allow: "*",
-  read_only: "yes"
-  }
-  - {
-  name: "videolan",
-  path: "/pubftp/videolan",
-  comment: "VideoLAN repository",
-  uid: "nobody",
-  gid: "nogroup",
-  hosts_allow: "*",
-  read_only: "yes"
-  }
 
 loc_nginx:
   service_name: ftp
   ssl: []
   servers:
     - server_name:
-        - "ftp"
-        - "ftp.*"
         - "mirror"
         - "mirror.*"
-        - "archive.ubuntu.com"
+      root: "/pool/mirror/pub"
-        - "fr.archive.ubuntu.com"
-        - "security.ubuntu.com"
-        - "ftps"
-        - "ftps.*"
-      root: "/pubftp"
       locations:
         - filter: "/"
           params:
-          - "autoindex on"
+            - "autoindex on"
-          - "autoindex_exact_size off"
+            - "autoindex_exact_size off"
-          - "add_before_body /.html/HEADER.html"
+            - "add_before_body /.html/HEADER.html"
-          - "add_after_body /.html/FOOTER.html"
+            - "add_after_body /.html/FOOTER.html"
-        - filter: "/pub/events/"
+    - server_name:
+        - "ftp"
+        - "ftp.*"
+        - "ftps"
+        - "ftps.*"
+      root: /pool/memorial
+      locations:
+        - filter: "/"
+          params:
+            - "autoindex on"
+            - "autoindex_exact_size off"
+            - "add_before_body /.html/HEADER.html"
+            - "add_after_body /.html/FOOTER.html"
+        - filter: ~ ^(\/pub)?(\/debian|\/ubuntu|\/videolan)(.*)$
+          params:
+            - return 301 http://mirror.crans.org$2$3
+        - filter: "/events"
           params:
             - "mp4"
             - "mp4_buffer_size 1m"
             - "mp4_max_buffer_size 5m"
+          

host_vars/tealc.adm.crans.org.yml

@@ -62,3 +62,19 @@ loc_mtail:
     - dhcpd.mtail
   remove:
     - radiusd.mtail
+
+loc_nginx:
+  service_name: ftp
+  ssl: []
+  servers:
+    - server_name:
+        - "mirror2"
+        - "mirror2.*"
+      root: "/pool/mirror/pub"
+      locations:
+        - filter: "/"
+          params:
+          - "autoindex on"
+          - "autoindex_exact_size off"
+          - "add_before_body /.html/HEADER.html"
+          - "add_after_body /.html/FOOTER.html"

hosts

@@ -31,6 +31,7 @@ belenios.adm.crans.org
 routeur-gulp.cachan-adm.crans.org
 
 [certbot]
+charybde.adm.crans.org
 sputnik.adm.crans.org
 
 [certbot:children]
@@ -113,12 +114,20 @@ tealc.adm.crans.org
 monitoring.adm.crans.org
 fyre.cachan-adm.crans.org
 
+[mirror_backend]
+charybde.adm.crans.org
+eclat.adm.crans.org
+
+[mirror_frontend]
+charybde.adm.crans.org
+tealc.adm.crans.org
+
 [nginx:children]
 django_cas
 galene
 jitsi
 mailman
-ntp_server
+mirror_frontend
 re2o_front
 reverseproxy
 roundcube
@@ -127,6 +136,7 @@ wiki
 
 [ntp_server]
 charybde.adm.crans.org
+eclat.adm.crans.org
 terenez.cachan-adm.crans.org
 
 [opendkim:children]

roles/apt-mirror/tasks/main.yml

@@ -0,0 +1,41 @@
+---
+- name: Install apt-mirror
+  apt:
+    update_cache: true
+    name: apt-mirror
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
+
+- name: Create mirrors directory
+  file:
+    path: "{{ apt_mirror.root }}/{{ item.host }}"
+    owner: apt-mirror
+    group: mirror
+    mode: 0755
+    state: directory
+  loop: "{{ apt_mirror.targets }}"
+
+- name: Create mirror symlink
+  file:
+    src: "{{ apt_mirror.root }}/{{ item.host }}/{{ item.symlink }}"
+    dest: "{{ apt_mirror.root }}/{{ item.name }}"
+    mode: 0755
+    state: link
+  loop: "{{ apt_mirror.targets }}"
+
+- name: Copy apt-mirror configurations
+  template:
+    src: apt/mirror.list.j2
+    dest: "/etc/apt/mirror.list"
+
+- name: Configure apt-mirror cron
+  template:
+    src: cron.d/apt-mirror.j2
+    dest: /etc/cron.d/apt-mirror
+
+- name: Indicate role in motd
+  template:
+    src: update-motd.d/05-service.j2
+    dest: /etc/update-motd.d/05-apt-mirror
+    mode: 0755

roles/apt-mirror/templates/apt/mirror.list.j2

@@ -0,0 +1,22 @@
+############# config ##################
+#
+# set base_path    /var/spool/apt-mirror
+#
+set mirror_path  {{ apt_mirror.root }}
+# set skel_path    $base_path/skel
+# set var_path     $base_path/var
+# set cleanscript $var_path/clean.sh
+# set defaultarch  <running host architecture>
+# set postmirror_script $var_path/postmirror.sh
+# set run_postmirror 0
+set nthreads     20
+set _tilde 0
+#
+############# end config ##############
+
+{% for target in apt_mirror.targets %}
+deb {{ target.scheme }}://{{ target.host }}/{{ target.path }} {{ target.suite }}{% for component in target.components %} {{ component }}{% endfor %}
+
+clean {{ target.scheme }}://{{ target.host }}/{{ target.path }}
+
+{% endfor %}

roles/apt-mirror/templates/cron.d/apt-mirror.j2

@@ -0,0 +1,4 @@
+#
+# Regular cron jobs for the apt-mirror package
+#
+0 4 * * *   apt-mirror  /usr/bin/apt-mirror > /var/spool/apt-mirror/var/cron.log

roles/apt-mirror/templates/update-motd.d/05-service.j2

@@ -0,0 +1,3 @@
+#!/usr/bin/tail +14
+{{ ansible_header | comment }}
+> pt-mirror a été déployé sur cette machine. Voir /etc/apt/mirror.list.

roles/ftpsync/tasks/main.yml

@@ -16,11 +16,31 @@
     home: /var/mirror  # unused, should be something empty
     shell: /bin/false
 
+- name: Create /etc/ftpsync directory
+  file:
+    path: "{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+  loop:
+    - "/etc/ftpsync"
+    - "{{ ftpsync.root }}/.html"
+
+- name: Create root directory
+  file:
+    path: "{{ ftpsync.root }}"
+    owner: mirror
+    group: mirror
+    mode: 0755
+    state: directory
+
+
 - name: Copy ftpsync configurations
   template:
     src: ftpsync.conf.j2
     dest: "/etc/ftpsync/ftpsync-{{ item.name }}.conf"
-  loop: "{{ ftpsync }}"
+  loop: "{{ ftpsync.targets }}"
 
 - name: Configure ftpsync cron
   template:
@@ -35,13 +55,10 @@
 
 - name: Copy configuration files
   template:
-    src: "{{ item.src }}"
+    src: "html/{{ item }}.j2"
-    dest: "{{ item.dest }}"
+    dest: "{{ ftpsync.root }}/.html/{{ item }}"
     mode: 0644
   loop:
-    - src: html/HEADER.html.j2
+    - HEADER.html
-      dest: /pubftp/.html/HEADER.html
+    - FOOTER.html
-    - src: html/FOOTER.html.j2
+    - style.min.css
-      dest: /pubftp/.html/FOOTER.html
-    - src: html/style.min.css.j2
-      dest: /pubftp/.html/style.min.css

roles/ftpsync/templates/ftpsync-cron.j2

@@ -1,4 +1,4 @@
 {{ ansible_header | comment }}
-{% for i in ftpsync %}
+{% for target in ftpsync.targets %}
-{{ i.cron_time }}  * * *   mirror ftpsync sync:archive:{{ i.name }}
+{{ target.cron_time }}  * * *   mirror ftpsync sync:archive:{{ target.name }}
 {% endfor %}

roles/ftpsync/templates/ftpsync.conf.j2

@@ -1,7 +1,7 @@
 {{ ansible_header | comment }}
 
-MIRRORNAME="CRANS"
+MIRRORNAME="{{ ftpsync.mirror.name }}"
-TO="{{ item.dest }}"
+TO="{{ ftpsync.root}}/{{ item.dest }}/"
 MAILTO="root"
 # HUB=false
 
@@ -18,10 +18,9 @@ RSYNC_PATH="{{ item.rsync_path }}"
 ## Mirror information options
 ########################################################################
 
-INFO_MAINTAINER="Les Nounous <contact@crans.org>"
+INFO_MAINTAINER="{{ ftpsync.mirror.info.maintainer }}"
-# INFO_SPONSOR="Example <https://example.com>"
+INFO_COUNTRY="{{ ftpsync.mirror.info.country }}"
-INFO_COUNTRY=FR
+INFO_LOCATION="{{ ftpsync.mirror.info.location }}"
-INFO_LOCATION="Cachan, Île-de-France"
 INFO_THROUGHPUT=1Gb
 
 ########################################################################

roles/rsync-mirror/tasks/main.yml

@@ -6,6 +6,23 @@
     home: /var/mirror  # unused, should be something empty
     shell: /bin/false
 
+- name: Create root directory
+  file:
+    path: "{{ ftpsync.root }}"
+    owner: mirror
+    group: mirror
+    mode: 0755
+    state: directory
+
+- name: Create target directory
+  file:
+    path: "{{ rsync_mirror.root }}/{{ item.dest }}"
+    owner: mirror
+    group: mirror
+    mode: 0755
+    state: directory
+  loop: "{{ rsync_mirror.targets }}"
+
 - name: Configure rsync cron
   template:
     src: rsync-mirror-cron.j2

roles/rsync-mirror/templates/rsync-mirror-cron.j2

@@ -1,4 +1,4 @@
 {{ ansible_header | comment }}
-{% for i in rsync_mirror %}
+{% for i in rsync_mirror.targets %}
-{{ i.cron_time }}  * * *   mirror rsync --times --links --hard-links --partial --block-size=8192 --recursive --exclude *-dvd-* --exclude source/ --verbose --stats --delete-after rsync://{{ i.rsync_host }}/{{ i.rsync_path }} {{ i.dest }} | tee -a "/var/log/mirror/{{ i.name }}.log" > /dev/null
+{{ i.cron_time }}  * * *   mirror rsync --times --links --hard-links --partial --block-size=8192 --recursive --exclude *-dvd-* --exclude source/ --verbose --stats --delete-after {%for exclude in i.exclude | default([]) %}--exclude {{ exclude }} {% endfor %}rsync://{{ i.rsync_host }}/{{ i.rsync_path }} {{ rsync_mirror.root }}/{{ i.dest }} | tee -a "/var/log/mirror/{{ i.name }}.log" > /dev/null
 {% endfor %}