diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md new file mode 100644 index 0000000..b04a12f --- /dev/null +++ b/hosts/physiques/cephiroth/ceph.md @@ -0,0 +1,143 @@ +# Ceph + +Ce fichier explique les tests et etapes effectue avec ceph. + + + ++ Déployer une conf minimal avec ceph et `services.ceph.global` deploye. ++ Creer une clé pour le cluster +```bash +sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' +``` ++ Creer un client admin +```bash +sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' +``` ++ Creer un bootstrap osd +```bash +sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r' +``` ++ On ajoute les cle generer au keyring +```bash +sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring +sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring +``` +(ca copie juste les deux clé précédente dans `/tmp/ceph.mon.keyring`) ++ on donne le keyring a l'user ceph +```bash +sudo chown ceph:ceph /tmp/ceph.mon.keyring +``` ++ Créer une monitor map +```bash +monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap +``` ++ Créer un dossier pour le monitor +```bash +sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname} +``` ++ On ajoute la monitor map et le keyring au monitor +```bash +sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring +``` + +On check si le service monitor start bien + +Pour voir le status du cluster +```bash +sudo ceph -s +``` + + +# Dashboard +- check si le module dashboard est activé +``` +sudo ceph mgr module ls +``` +- pour activer si pas activer +```bash +ceph mgr module enable dashboard +``` + +Dans un premier temps pour le test j'ai desactiver le ssl : +```bash +ceph config set mgr mgr/dashboard/ssl false +``` + +Pour voir sur quoi le dashboard ecoute (apres avoir desactive ssl) +```bash +sudo ceph mgr services +``` + +Pour les user check https://docs.ceph.com/en/latest/mgr/dashboard/#username-and-password + +# OSD + +Pour créer un osd (attention de prendre le bon disque parce que ca supprime les données du disque) : +```bash +sudo ceph-volume lvm create --data /dev/sdf --no-systemd +``` + +# Problèmes + +Si avec `ceph -s` vous avez : + +`1 monitors have not enabled msgr2`, +activez le msgr2, afin que les vms proxmox puissent utiliser les disques : +```bash +sudo ceph mon enable-msgr2 +``` +`mons are allowing insecure global_id reclaim` +alors +```bash +ceph config set mon auth_allow_insecure_global_id_reclaim false +``` + + +# Info + +- Monitor : c'est lui qui garde la map d'ou est quoi +- Manager : il check si tout va bien, c'est lui qui a le dashboard +- OSD : ca gere 1 disque physique + + +# Proposition Architecture + +## CephFS +Il semble qu'il est préférable d'avoir 1 cephfs avec des sous répertoire. +La raison principale est qu'il faut 1 mds/cephfs, et ca semble relativement +gourmand en ressources. + +- cephfs + - home-nounou + - home-adh + - mail + - mirror + +> Note: on pourrait penser bouger le mirror vers du S3 ? + +## RBD (raw block devices): +- vm-adh +- vm-nounou + + +# Question en suspend +- backup ? +- postgresql +- monitoring + - il y a du prometheus et des dashboard grafana ca devrait le faire +- user : + - 1 user/"volume" ou 1user/host, autres ? +- migration nfs -> cephfs ? + +# Plan Migration + + + +# Monitoring + +Les dashboard grafana viennent de : + +Pour activer prometheus (port 9283 par defaut) : +```bash +ceph mgr module enable prometheus +``` diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix new file mode 100644 index 0000000..8d72a15 --- /dev/null +++ b/hosts/physiques/cephiroth/ceph.nix @@ -0,0 +1,85 @@ +{ pkgs, ... }: + +{ + # La doc ceph : https://docs.ceph.com/en/quincy/install/manual-deployment/ + + services.ceph = { + enable = true; + + # osd, 1 deamon/disk avec un id unique dans le cluster. + osd = { + enable = true; + + # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine + # les machines physique ont des id < 100, donc ca fonctionne + daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"]; + extraConfig = { + "public_addr" = "172.16.4.3"; + "cluster_addr" = "172.16.6.3"; + }; + }; + + # monitor, garde la map du cluster + mon = { + enable = true; + extraConfig = { + "mon host" = "172.16.4.3"; + "mon initial members" = "cephiroth"; + }; + daemons = ["cephiroth"]; + }; + + # Dashboard + mgr = { + enable = true; + daemons = ["cephiroth"]; + extraConfig = { + "mgr_initial_modules" = "dashboard prometheus"; + # TODO: setup prometheus listen addr + # le port de prometheus est 9283 pour ceph + }; + }; + + # pour cephFS + mds = { + enable = true; + daemons = ["cephiroth"]; + }; + + global = { + # Comme nom de cluster on choisit `ceph` (la valeur par défaut), + # car on aura qu'un cluster. + clusterName = "ceph"; + # fsid unique par node, généré avec `uuidgen` + fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0"; + monHost = "172.16.4.3"; + monInitialMembers = "cephiroth"; + publicNetwork = "172.16.4.0/24"; + clusterNetwork = "172.16.6.0/24"; + }; + + client = { + enable = true; + }; + }; + + # automaticaly start osd when we restart the server + systemd.services.ceph-mesh = { + enable = true; + description = "Ceph OSD Bindings"; + unitConfig = { + After = "local-fs.target"; + Wants = "local-fs.target"; + }; + serviceConfig = { + Type = "oneshot"; + KillMode = "none"; + Environment = "CEPH_VOLUME_TIMEOUT=10000 PATH=$PATH:/run/current-system/sw/bin/"; + ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT /run/current-system/sw/bin/ceph-volume lvm activate --all --no-systemd'"; + TimeoutSec = 0; + }; + wantedBy = ["multi-user.target"]; + }; + + environment.systemPackages = [ pkgs.ceph pkgs.smartmontools ]; +} diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix new file mode 100644 index 0000000..f4c727a --- /dev/null +++ b/hosts/physiques/cephiroth/default.nix @@ -0,0 +1,58 @@ +{ pkgs, lib, ... }: + +{ + imports = [ + ./hardware-configuration.nix + ./networking.nix + ./ceph.nix + ]; + + networking.hostId = "bbdd1133"; + networking.hostName = "cephiroth"; + + boot.loader.grub = { + enable = true; + efiSupport = true; + zfsSupport = true; + device = "nodev"; + mirroredBoots = [ + # { + # devices = ["/dev/sdi"]; + # path = "/boot"; + # } + # { + # devices = [ "/dev/sdj" ]; + # path = "/boot-fallback"; + # } + { + devices = ["nodev"]; + path = "/boot-fallback"; + } + ]; + }; + + boot.tmp = { + useTmpfs = true; + cleanOnBoot = true; + }; + + boot.supportedFilesystems = [ "zfs" ]; + boot.zfs.extraPools = ["head"]; + boot.loader.efi.canTouchEfiVariables = true; + + + crans = { + enable = true; + + networking = { + id = 3; + # On doit utiliser des vlans et du bond, donc + # on desactive la config network des vms + enable = lib.mkForce false; + }; + resticClient.enable = false; + }; + + + system.stateVersion = "24.05"; +} diff --git a/hosts/physiques/cephiroth/hardware-configuration.nix b/hosts/physiques/cephiroth/hardware-configuration.nix new file mode 100644 index 0000000..527881a --- /dev/null +++ b/hosts/physiques/cephiroth/hardware-configuration.nix @@ -0,0 +1,58 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "uhci_hcd" "hpsa" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "head/root"; + fsType = "zfs"; + options = ["zfsutil"]; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/C900-92D1"; + fsType = "vfat"; + options = [ "fmask=0022" "dmask=0022" ]; + }; + + fileSystems."/boot-fallback" = + { device = "/dev/disk/by-uuid/C931-84A4"; + fsType = "vfat"; + options = [ "fmask=0022" "dmask=0022" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.10.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.3.useDHCP = lib.mkDefault true; + # networking.interfaces.bonding_masters.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.eno2.useDHCP = lib.mkDefault true; + # networking.interfaces.eno3.useDHCP = lib.mkDefault true; + # networking.interfaces.eno4.useDHCP = lib.mkDefault true; + # networking.interfaces.eno49.useDHCP = lib.mkDefault true; + # networking.interfaces.eno50.useDHCP = lib.mkDefault true; + # networking.interfaces.eno51.useDHCP = lib.mkDefault true; + # networking.interfaces.eno52.useDHCP = lib.mkDefault true; + # networking.interfaces.enp11s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp11s0f1.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix new file mode 100644 index 0000000..027488a --- /dev/null +++ b/hosts/physiques/cephiroth/networking.nix @@ -0,0 +1,86 @@ +{ ... }: + +{ + networking = { + + dhcpcd.enable = false; + firewall.enable = false; + + vlans = { + srv-nat = { + id = 3; + interface = "bond0"; + }; + san = { + id = 4; + interface = "bond0"; + }; + ceph = { + id = 6; + interface = "bond0"; + }; + adm = { + id = 10; + interface = "bond0"; + }; + }; + + bonds.bond0 = { + interfaces = [ "ens2f0" "ens2f1" ]; + driverOptions.mode = "802.3ad"; + }; + + interfaces = { + ens2f0.mtu = 9000; + ens2f1.mtu = 9000; + + bonds.mtu = 9000; + + srv-nat = { + ipv4 = { + addresses = [ + { + address = "172.16.3.3"; + prefixLength = 24; + } + ]; + }; + }; + ceph = { + mtu = 9000; + ipv4 = { + addresses = [ + { + address = "172.16.6.3"; + prefixLength = 24; + } + ]; + }; + }; + adm = { + mtu = 9000; + ipv4 = { + addresses = [ + { + address = "172.16.10.3"; + prefixLength = 24; + } + ]; + }; + }; + san = { + mtu = 9000; + ipv4 = { + addresses = [ + { + address = "172.16.4.3"; + prefixLength = 24; + } + ]; + }; + }; + }; + defaultGateway = "172.16.3.99"; + nameservers = [ "172.16.10.128" ]; + }; +}