From f45f8973d44ff344a552f586a38d9a6c3dfb4ba4 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Mon, 26 Jan 2026 20:02:39 +0100 Subject: [PATCH 01/32] cette fois ce sera la bonne ? --- flake.nix | 5 ++ hosts/physiques/cephiroth/default.nix | 49 ++++++++++++++ .../cephiroth/hardware-configuration.nix | 66 +++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 hosts/physiques/cephiroth/default.nix create mode 100644 hosts/physiques/cephiroth/hardware-configuration.nix diff --git a/flake.nix b/flake.nix index d841f39..820c1c9 100644 --- a/flake.nix +++ b/flake.nix @@ -44,6 +44,11 @@ specialArgs = inputs; modules = [ ./hosts/vm/apprentix ] ++ baseModules; }; + + cephiroth = nixosSystem { + specialArgs = inputs; + modules = [ ./hosts/physiques/cephiroth ] ++ baseModules; + }; jitsi = nixosSystem { specialArgs = inputs; diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix new file mode 100644 index 0000000..e660681 --- /dev/null +++ b/hosts/physiques/cephiroth/default.nix @@ -0,0 +1,49 @@ +{ pkgs, ... }: + +{ + imports = [ + ./hardware-configuration.nix + ]; + + networking.hostId = "121cdcb4"; + networking.hostName = "cephiroth"; + + boot.loader.grub.enable = true; + boot.loader.grub.efiSupport = true; + boot.loader.grub.device = "nodev"; + + boot.loader.grub.mirroredBoots = [ + { + devices = [ "nodev" ]; + path = "/boot-fallback"; + } + ]; + + + crans = { + enable = true; + + networking = { + id = 3; + adm = { + enable = true; + interface = "bond0"; + }; + srvNat = { + enable = true; + interface = "bond0"; + }; + }; + resticClient.enable = false; + }; + + + networking = { + bonds.bond0 = { + interfaces = [ "enp11s0f0" "enp11s0f1" ]; + driverOptions.mode = "802.3ad"; + }; + }; + + system.stateVersion = "24.05"; +} diff --git a/hosts/physiques/cephiroth/hardware-configuration.nix b/hosts/physiques/cephiroth/hardware-configuration.nix new file mode 100644 index 0000000..c04f8b9 --- /dev/null +++ b/hosts/physiques/cephiroth/hardware-configuration.nix @@ -0,0 +1,66 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ + config, + lib, + pkgs, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ + "ehci_pci" + "ata_piix" + "uhci_hcd" + "hpsa" + "usbhid" + "usb_storage" + "sd_mod" + "sr_mod" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/backups" = { + device = "backups"; + fsType = "zfs"; + }; + + fileSystems."/" = { + device = "head"; + fsType = "zfs"; + }; + + fileSystems."/boot" = { + device = "/dev/disk/by-uuid/8E4A-8E9F"; + fsType = "vfat"; + options = [ + "fmask=0022" + "dmask=0022" + ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.eno2.useDHCP = lib.mkDefault true; + # networking.interfaces.eno3.useDHCP = lib.mkDefault true; + # networking.interfaces.eno4.useDHCP = lib.mkDefault true; + # networking.interfaces.enp4s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp4s0f1.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} From d56fd3053b18760beb720cdff471a5de2c969b71 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Mon, 26 Jan 2026 22:05:31 +0100 Subject: [PATCH 02/32] avec un disque de boot c'est mieux ? --- hosts/physiques/cephiroth/default.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index e660681..02d4b2f 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -14,7 +14,11 @@ boot.loader.grub.mirroredBoots = [ { - devices = [ "nodev" ]; + devices = ["/dev/sdi"]; + path = "/boot"; + } + { + devices = [ "/dev/sdj" ]; path = "/boot-fallback"; } ]; From 01b78dd455c717c3154761b1ef2fbd5f53d51cd6 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Mon, 26 Jan 2026 22:18:04 +0100 Subject: [PATCH 03/32] Et avec le bon hardware config ? --- .../cephiroth/hardware-configuration.nix | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/hosts/physiques/cephiroth/hardware-configuration.nix b/hosts/physiques/cephiroth/hardware-configuration.nix index c04f8b9..84c49b8 100644 --- a/hosts/physiques/cephiroth/hardware-configuration.nix +++ b/hosts/physiques/cephiroth/hardware-configuration.nix @@ -1,51 +1,34 @@ # Do not modify this file! It was generated by ‘nixos-generate-config’ # and may be overwritten by future invocations. Please make changes # to /etc/nixos/configuration.nix instead. -{ - config, - lib, - pkgs, - modulesPath, - ... -}: +{ config, lib, pkgs, modulesPath, ... }: { - imports = [ - (modulesPath + "/installer/scan/not-detected.nix") - ]; + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; - boot.initrd.availableKernelModules = [ - "ehci_pci" - "ata_piix" - "uhci_hcd" - "hpsa" - "usbhid" - "usb_storage" - "sd_mod" - "sr_mod" - ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "uhci_hcd" "hpsa" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-intel" ]; boot.extraModulePackages = [ ]; - fileSystems."/backups" = { - device = "backups"; - fsType = "zfs"; - }; + fileSystems."/" = + { device = "head/root"; + fsType = "zfs"; + }; - fileSystems."/" = { - device = "head"; - fsType = "zfs"; - }; + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/C900-92D1"; + fsType = "vfat"; + options = [ "fmask=0022" "dmask=0022" ]; + }; - fileSystems."/boot" = { - device = "/dev/disk/by-uuid/8E4A-8E9F"; - fsType = "vfat"; - options = [ - "fmask=0022" - "dmask=0022" - ]; - }; + fileSystems."/boot-fallback" = + { device = "/dev/disk/by-uuid/C931-84A4"; + fsType = "vfat"; + options = [ "fmask=0022" "dmask=0022" ]; + }; swapDevices = [ ]; @@ -53,13 +36,21 @@ # (the default) this is the recommended approach. When using systemd-networkd it's # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - # networking.useDHCP = lib.mkDefault true; + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.10.useDHCP = lib.mkDefault true; + # networking.interfaces.bond0.3.useDHCP = lib.mkDefault true; + # networking.interfaces.bonding_masters.useDHCP = lib.mkDefault true; # networking.interfaces.eno1.useDHCP = lib.mkDefault true; # networking.interfaces.eno2.useDHCP = lib.mkDefault true; # networking.interfaces.eno3.useDHCP = lib.mkDefault true; # networking.interfaces.eno4.useDHCP = lib.mkDefault true; - # networking.interfaces.enp4s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp4s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.eno49.useDHCP = lib.mkDefault true; + # networking.interfaces.eno50.useDHCP = lib.mkDefault true; + # networking.interfaces.eno51.useDHCP = lib.mkDefault true; + # networking.interfaces.eno52.useDHCP = lib.mkDefault true; + # networking.interfaces.enp11s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp11s0f1.useDHCP = lib.mkDefault true; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; From a034ba29554b917904753a688cec004f92abe8a4 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Mon, 26 Jan 2026 23:04:46 +0100 Subject: [PATCH 04/32] use old networking file --- hosts/physiques/cephiroth/default.nix | 16 +------- hosts/physiques/cephiroth/networking.nix | 49 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 hosts/physiques/cephiroth/networking.nix diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index 02d4b2f..e5b0055 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -3,6 +3,7 @@ { imports = [ ./hardware-configuration.nix + ./networking.nix ]; networking.hostId = "121cdcb4"; @@ -29,25 +30,10 @@ networking = { id = 3; - adm = { - enable = true; - interface = "bond0"; - }; - srvNat = { - enable = true; - interface = "bond0"; - }; }; resticClient.enable = false; }; - networking = { - bonds.bond0 = { - interfaces = [ "enp11s0f0" "enp11s0f1" ]; - driverOptions.mode = "802.3ad"; - }; - }; - system.stateVersion = "24.05"; } diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix new file mode 100644 index 0000000..c1b540b --- /dev/null +++ b/hosts/physiques/cephiroth/networking.nix @@ -0,0 +1,49 @@ +{ ... }: + +{ + networking = { + + dhcpcd.enable = false; + + vlans = { + srv-nat = { + id = 3; + interface = "bond0"; + }; + adm = { + id = 10; + interface = "bond0"; + }; + }; + + bonds.bond0 = { + interfaces = [ "enp11s0f0" "enp11s0f1" ]; + driverOptions.mode = "802.3ad"; + }; + + interfaces = { + srv-nat = { + ipv4 = { + addresses = [ + { + address = "172.16.3.3"; + prefixLength = 24; + } + ]; + }; + }; + adm = { + ipv4 = { + addresses = [ + { + address = "172.16.10.3"; + prefixLength = 24; + } + ]; + }; + }; + }; + defaultGateway = "172.16.3.99"; + nameservers = [ "172.16.10.128" ]; + }; +} From e95fc3000bdff0676d9e8530fadb387389d15071 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Mon, 26 Jan 2026 23:33:26 +0100 Subject: [PATCH 05/32] test... --- hosts/physiques/cephiroth/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index e5b0055..cf0f0be 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -14,10 +14,10 @@ boot.loader.grub.device = "nodev"; boot.loader.grub.mirroredBoots = [ - { - devices = ["/dev/sdi"]; - path = "/boot"; - } + # { + # devices = ["/dev/sdi"]; + # path = "/boot"; + # } { devices = [ "/dev/sdj" ]; path = "/boot-fallback"; From 898b425a1d6d1466a807a3329e823827f4ac19ee Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Tue, 27 Jan 2026 09:39:51 +0100 Subject: [PATCH 06/32] et maintenant ? --- hosts/physiques/cephiroth/default.nix | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index cf0f0be..196ed0a 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -9,20 +9,22 @@ networking.hostId = "121cdcb4"; networking.hostName = "cephiroth"; - boot.loader.grub.enable = true; - boot.loader.grub.efiSupport = true; - boot.loader.grub.device = "nodev"; - - boot.loader.grub.mirroredBoots = [ - # { - # devices = ["/dev/sdi"]; - # path = "/boot"; - # } - { - devices = [ "/dev/sdj" ]; - path = "/boot-fallback"; - } - ]; + boot.loader.grub = { + enable = true; + efiSupport = true; + device = "nodev"; + mirroredBoots = [ + # { + # devices = ["/dev/sdi"]; + # path = "/boot"; + # } + { + devices = [ "/dev/sdj" ]; + path = "/boot-fallback"; + } + ]; + devices = ["/dev/sdi" "/dev/sdj"]; + }; crans = { From 2facd0baf40d3d55fc9881c99771a30f7f1cd335 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Tue, 27 Jan 2026 12:12:23 +0100 Subject: [PATCH 07/32] on y croit --- hosts/physiques/cephiroth/default.nix | 14 +++++++++++--- hosts/physiques/cephiroth/networking.nix | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index 196ed0a..d82fa5b 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -6,7 +6,7 @@ ./networking.nix ]; - networking.hostId = "121cdcb4"; + networking.hostId = "bbdd1133"; networking.hostName = "cephiroth"; boot.loader.grub = { @@ -18,12 +18,20 @@ # devices = ["/dev/sdi"]; # path = "/boot"; # } + # { + # devices = [ "/dev/sdj" ]; + # path = "/boot-fallback"; + # } { - devices = [ "/dev/sdj" ]; + devices = ["nodev"]; path = "/boot-fallback"; } ]; - devices = ["/dev/sdi" "/dev/sdj"]; + }; + + boot.tmp = { + useTmpfs = true; + cleanOnBoot = true; }; diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index c1b540b..d43206f 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -10,6 +10,10 @@ id = 3; interface = "bond0"; }; + ceph = { + id = 6; + interface = "bond0"; + }; adm = { id = 10; interface = "bond0"; @@ -22,6 +26,9 @@ }; interfaces = { + enp11s0f0.mtu = 9000; + enp11s0f1.mtu = 9000; + srv-nat = { ipv4 = { addresses = [ @@ -32,6 +39,16 @@ ]; }; }; + ceph = { + ipv4 = { + addresses = [ + { + address = "172.16.6.3"; + prefixLength = 24; + } + ]; + }; + }; adm = { ipv4 = { addresses = [ From 854675141140f44b5125608a972211e009cebb6a Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Wed, 28 Jan 2026 20:20:45 +0100 Subject: [PATCH 08/32] un nouveau test --- hosts/physiques/cephiroth/default.nix | 10 +++++++++- hosts/physiques/cephiroth/hardware-configuration.nix | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index d82fa5b..7ee395e 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -1,4 +1,4 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: { imports = [ @@ -12,6 +12,7 @@ boot.loader.grub = { enable = true; efiSupport = true; + zfsSupport = true; device = "nodev"; mirroredBoots = [ # { @@ -34,12 +35,19 @@ cleanOnBoot = true; }; + boot.supportedFilesystems = [ "zfs" ]; + boot.zfs.extraPools = ["head"]; + boot.loader.efi.canTouchEfiVariables = true; + crans = { enable = true; networking = { id = 3; + # On doit utiliser des vlans et du bond, donc + # on desactive la config network des vms + enable = lib.mkForce false; }; resticClient.enable = false; }; diff --git a/hosts/physiques/cephiroth/hardware-configuration.nix b/hosts/physiques/cephiroth/hardware-configuration.nix index 84c49b8..527881a 100644 --- a/hosts/physiques/cephiroth/hardware-configuration.nix +++ b/hosts/physiques/cephiroth/hardware-configuration.nix @@ -16,6 +16,7 @@ fileSystems."/" = { device = "head/root"; fsType = "zfs"; + options = ["zfsutil"]; }; fileSystems."/boot" = From 1d0332cdcc2f4ec5aa860bea004500bd92c6e7d0 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Wed, 28 Jan 2026 21:53:59 +0100 Subject: [PATCH 09/32] les interfaces reseaux n'ont plus le meme nom... --- flake.lock | 18 +++++++++--------- hosts/physiques/cephiroth/networking.nix | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 6d0a2ed..cda05d5 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1765835352, - "narHash": "sha256-XswHlK/Qtjasvhd1nOa1e8MgZ8GS//jBoTqWtrS1Giw=", + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "a34fae9c08a15ad73f295041fec82323541400a9", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", "type": "github" }, "original": { @@ -86,11 +86,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1765838191, - "narHash": "sha256-m5KWt1nOm76ILk/JSCxBM4MfK3rYY7Wq9/TZIIeGnT8=", + "lastModified": 1769318308, + "narHash": "sha256-Mjx6p96Pkefks3+aA+72lu1xVehb6mv2yTUUqmSet6Q=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c6f52ebd45e5925c188d1a20119978aa4ffd5ef6", + "rev": "1cd347bf3355fce6c64ab37d3967b4a2cb4b878c", "type": "github" }, "original": { @@ -145,11 +145,11 @@ ] }, "locked": { - "lastModified": 1766000401, - "narHash": "sha256-+cqN4PJz9y0JQXfAK5J1drd0U05D5fcAGhzhfVrDlsI=", + "lastModified": 1769515380, + "narHash": "sha256-CWWK3PaQ7zhr+Jcf5zyaTR2cfRBXPo09H7+5nWApL8s=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "42d96e75aa56a3f70cab7e7dc4a32868db28e8fd", + "rev": "9911802c2822def2eec3d22e2cafd1619ede94a5", "type": "github" }, "original": { diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index d43206f..3b95a1d 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -21,7 +21,7 @@ }; bonds.bond0 = { - interfaces = [ "enp11s0f0" "enp11s0f1" ]; + interfaces = [ "ens2f0" "ens2f1" ]; driverOptions.mode = "802.3ad"; }; From e2dfe2aa86cad419254c331adc5fb36e8a3c2f5d Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 17:55:17 +0100 Subject: [PATCH 10/32] commencons a jouer avec ceph --- hosts/physiques/cephiroth/ceph.nix | 44 +++++++++++++++++++++++++++ hosts/physiques/cephiroth/default.nix | 1 + 2 files changed, 45 insertions(+) create mode 100644 hosts/physiques/cephiroth/ceph.nix diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix new file mode 100644 index 0000000..6ba44f5 --- /dev/null +++ b/hosts/physiques/cephiroth/ceph.nix @@ -0,0 +1,44 @@ +{ pkgs, ... }: + +{ + # La doc ceph : https://docs.ceph.com/en/quincy/install/manual-deployment/ + + services.ceph = { + enable = true; + + # osd, 1 deamon/disk avec un id unique dans le cluster. + osd = { + enable = true; + + # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine + # les machines physique ont des id < 100, donc ca fonctionne + daemons = ["103"]; + }; + + # monitor + mon = { + enable = false; + extraConfig = { + "mon host" = "172.16.6.3"; + "mon initial members" = "cephiroth"; + }; + daemons = ["1"]; + }; + + global = { + # Comme nom de cluster on choisit `ceph` (la valeur par défaut), + # car on aura qu'un cluster. + clusterName = "ceph"; + # fsid unique par node, généré avec `uuidgen` + fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0"; + monHost = "172.16.6.3"; + monInitialMembers = "cephiroth"; + }; + + client = { + enable = true; + }; + }; + + environment.systemPackages = [ pkgs.ceph ]; +} diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index 7ee395e..f4c727a 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -4,6 +4,7 @@ imports = [ ./hardware-configuration.nix ./networking.nix + ./ceph.nix ]; networking.hostId = "bbdd1133"; From 4ac21e0d4c42cfaec7bb3a01a2492f30809e2da1 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 18:19:30 +0100 Subject: [PATCH 11/32] enable monitor --- hosts/physiques/cephiroth/ceph.md | 41 ++++++++++++++++++++++++++++++ hosts/physiques/cephiroth/ceph.nix | 4 +-- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 hosts/physiques/cephiroth/ceph.md diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md new file mode 100644 index 0000000..6d59bbd --- /dev/null +++ b/hosts/physiques/cephiroth/ceph.md @@ -0,0 +1,41 @@ +# Ceph + +Ce fichier explique les tests et etapes effectue avec ceph. + + + ++ Déployer une conf minimal avec ceph et `services.ceph.global` deploye. ++ Creer une clé pour le cluster +```bash +sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' +``` ++ Creer un client admin +```bash +sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' +``` ++ Creer un bootstrap osd +```bash +sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r' +``` ++ On ajoute les cle generer au keyring +```bash +sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring +sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring +``` +(ca copie juste les deux clé précédente dans `/tmp/ceph.mon.keyring`) ++ on donne le keyring a l'user ceph +```bash +sudo chown ceph:ceph /tmp/ceph.mon.keyring +``` ++ Créer une monitor map +```bash +monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap +``` ++ Créer un dossier pour le monitor +```bash +sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname} +``` ++ On ajoute la monitor map et le keyring au monitor +```bash +sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring +``` diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 6ba44f5..28f77c6 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -17,12 +17,12 @@ # monitor mon = { - enable = false; + enable = true; extraConfig = { "mon host" = "172.16.6.3"; "mon initial members" = "cephiroth"; }; - daemons = ["1"]; + daemons = ["cephiroth"]; }; global = { From cd1befac9d11df65ec2f213e478a68966a08f0f6 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 18:34:56 +0100 Subject: [PATCH 12/32] add mgr --- hosts/physiques/cephiroth/ceph.md | 14 ++++++++++++++ hosts/physiques/cephiroth/ceph.nix | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md index 6d59bbd..9e1effc 100644 --- a/hosts/physiques/cephiroth/ceph.md +++ b/hosts/physiques/cephiroth/ceph.md @@ -39,3 +39,17 @@ sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname} ```bash sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring ``` + +On check si le service monitor start bien + +Pour voir le status du cluster +```bash +sudo ceph -s +``` + + +# Info + +- Monitor : c'est lui qui garde la map d'ou est quoi +- Manager : il check si tout va bien, c'est lui qui a le dashboard +- OSD : ca gere 1 disque physique diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 28f77c6..09828e1 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -15,7 +15,7 @@ daemons = ["103"]; }; - # monitor + # monitor, garde la map du cluster mon = { enable = true; extraConfig = { @@ -25,6 +25,15 @@ daemons = ["cephiroth"]; }; + # Dashboard + mgr = { + enable = true; + daemons = ["cephiroth"]; + extraConfig = { + "mgr_initial_modules" = ["dashboard"]; + }; + }; + global = { # Comme nom de cluster on choisit `ceph` (la valeur par défaut), # car on aura qu'un cluster. From de418773216fbe3506d59d34ada0fdc1d9e89f41 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 18:36:15 +0100 Subject: [PATCH 13/32] fix ? config mgr --- hosts/physiques/cephiroth/ceph.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 09828e1..f5d8c2e 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -30,7 +30,7 @@ enable = true; daemons = ["cephiroth"]; extraConfig = { - "mgr_initial_modules" = ["dashboard"]; + "mgr_initial_modules" = "dashboard"; }; }; From 331df8e03e08404aa31ca7a280083525ca7d00c8 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 19:06:47 +0100 Subject: [PATCH 14/32] fix: use id generated by ceph --- hosts/physiques/cephiroth/ceph.md | 29 +++++++++++++++++++++++++++++ hosts/physiques/cephiroth/ceph.nix | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md index 9e1effc..bca526b 100644 --- a/hosts/physiques/cephiroth/ceph.md +++ b/hosts/physiques/cephiroth/ceph.md @@ -48,6 +48,35 @@ sudo ceph -s ``` +# Dashboard +- check si le module dashboard est activé +``` +sudo ceph mgr module ls +``` +- pour activer si pas activer +```bash +ceph mgr module enable dashboard +``` + +Dans un premier temps pour le test j'ai desactiver le ssl : +```bash +ceph config set mgr mgr/dashboard/ssl false +``` + +Pour voir sur quoi le dashboard ecoute (apres avoir desactive ssl) +```bash +sudo ceph mgr services +``` + +Pour les user check https://docs.ceph.com/en/latest/mgr/dashboard/#username-and-password + +# OSD + +Pour créer un osd (attention de prendre le bon disque parce que ca supprime les données du disque) : +```bash +sudo ceph-volume lvm create --data /dev/sdf --no-systemd +``` + # Info - Monitor : c'est lui qui garde la map d'ou est quoi diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index f5d8c2e..ac32021 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -12,7 +12,7 @@ # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine # les machines physique ont des id < 100, donc ca fonctionne - daemons = ["103"]; + daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"]; }; # monitor, garde la map du cluster From 0e85064903f2ee3229054dac11782890cbdef8fc Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Thu, 29 Jan 2026 22:22:48 +0100 Subject: [PATCH 15/32] add mds for cephFS --- hosts/physiques/cephiroth/ceph.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index ac32021..123fa03 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -34,6 +34,12 @@ }; }; + # pour cephFS + mds = { + enable = true; + daemons = ["cephiroth"]; + }; + global = { # Comme nom de cluster on choisit `ceph` (la valeur par défaut), # car on aura qu'un cluster. From 12b268e63d33c3d8e55d074350c90d2ecf3f9b17 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Fri, 30 Jan 2026 00:29:54 +0100 Subject: [PATCH 16/32] add ceph vlan --- hosts/physiques/cephiroth/ceph.md | 11 +++++++++++ modules/crans/networking.nix | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md index bca526b..fc805c6 100644 --- a/hosts/physiques/cephiroth/ceph.md +++ b/hosts/physiques/cephiroth/ceph.md @@ -82,3 +82,14 @@ sudo ceph-volume lvm create --data /dev/sdf --no-systemd - Monitor : c'est lui qui garde la map d'ou est quoi - Manager : il check si tout va bien, c'est lui qui a le dashboard - OSD : ca gere 1 disque physique + + +# Proposition Architecture + +CephFS: +- home-adh +- home-nounou + +RBD (raw block devices): +- vm-adh +- vm-nounou diff --git a/modules/crans/networking.nix b/modules/crans/networking.nix index 60ac51e..622b826 100644 --- a/modules/crans/networking.nix +++ b/modules/crans/networking.nix @@ -37,6 +37,17 @@ in }; }; + ceph = { + enable = mkEnableOption "Configuration du VLAN ceph."; + + interface = mkOption { + type = types.str; + default = "ens20"; + example = "ens19"; + description = "Nom de l'interface réseau sur laquelle est située le VLAN ceph."; + }; + }; + srv = { enable = mkEnableOption "Configuration du VLAN srv."; @@ -114,6 +125,23 @@ in } ]; }; + + # Configuration du VLAN ceph + ceph = { + ipv4.addresses = [ + { + address = "172.16.6.${idString}"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "fd00::6:0:ff:fe0${isVmString}:${hostIdString}06"; + prefixLength = 64; + } + ]; + }; # Configuration du VLAN srv srv = { From 44b299d15fea98a93377e4e03e8154e709cf6dce Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Fri, 30 Jan 2026 20:14:16 +0100 Subject: [PATCH 17/32] fix: mtu network interface --- hosts/physiques/cephiroth/ceph.md | 29 ++++++++++++++++++++---- hosts/physiques/cephiroth/networking.nix | 4 ++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md index fc805c6..60076cc 100644 --- a/hosts/physiques/cephiroth/ceph.md +++ b/hosts/physiques/cephiroth/ceph.md @@ -86,10 +86,31 @@ sudo ceph-volume lvm create --data /dev/sdf --no-systemd # Proposition Architecture -CephFS: -- home-adh -- home-nounou +## CephFS +Il semble qu'il est préférable d'avoir 1 cephfs avec des sous répertoire. +La raison principale est qu'il faut 1 mds/cephfs, et ca semble relativement +gourmand en ressources. -RBD (raw block devices): +- cephfs + - home-nounou + - home-adh + - mail + - mirror + +> Note: on pourrait penser bouger le mirror vers du S3 ? + +## RBD (raw block devices): - vm-adh - vm-nounou + + +# Question en suspend +- backup ? +- postgresql +- monitoring + - il y a du prometheus et des dashboard grafana ca devrait le faire +- user : + - 1 user/"volume" ou 1user/host, autres ? +- migration nfs -> cephfs ? + +# Plan Migration diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 3b95a1d..900501b 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -26,8 +26,8 @@ }; interfaces = { - enp11s0f0.mtu = 9000; - enp11s0f1.mtu = 9000; + ens2f0.mtu = 9000; + ens2f1.mtu = 9000; srv-nat = { ipv4 = { From 929b5935c218c00fd72c71c31a54d8c53736c670 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Fri, 30 Jan 2026 20:41:58 +0100 Subject: [PATCH 18/32] add smartmontools to monitor disk --- hosts/physiques/cephiroth/ceph.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 123fa03..11792ec 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -55,5 +55,5 @@ }; }; - environment.systemPackages = [ pkgs.ceph ]; + environment.systemPackages = [ pkgs.ceph pkgs.smartmontools ]; } From 342cd2abc6cfc3b42ece608b327398fd46bad99e Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sat, 31 Jan 2026 13:51:25 +0100 Subject: [PATCH 19/32] auto start osd code from https://wiki.nixos.org/wiki/Ceph --- hosts/physiques/cephiroth/ceph.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 11792ec..0cbf089 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -55,5 +55,23 @@ }; }; + # automaticaly start osd when we restart the server + systemd.services.ceph-mesh = { + enable = true; + description = "Ceph OSD Bindings"; + unitConfig = { + After = "local-fs.target"; + Wants = "local-fs.target"; + }; + serviceConfig = { + Type = "oneshot"; + KillMode = "none"; + Environment = "CEPH_VOLUME_TIMEOUT=10000 PATH=$PATH:/run/current-system/sw/bin/"; + ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT /run/current-system/sw/bin/ceph-volume lvm activate --all --no-systemd'"; + TimeoutSec = 0; + }; + wantedBy = ["multi-user.target"]; + }; + environment.systemPackages = [ pkgs.ceph pkgs.smartmontools ]; } From 4625592db4f075d31ec4387b15d991efc235af73 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sat, 31 Jan 2026 22:36:25 +0100 Subject: [PATCH 20/32] add san vlan pour servir cephfs et san --- hosts/physiques/cephiroth/networking.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 900501b..15c7d40 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -59,6 +59,16 @@ ]; }; }; + san = { + ipv4 = { + addresses = [ + { + address = "172.16.4.3"; + prefixLength = 24; + } + ]; + }; + }; }; defaultGateway = "172.16.3.99"; nameservers = [ "172.16.10.128" ]; From f9f36e2593df4e68a4f4aa9aaeddcc47cedb453e Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sat, 31 Jan 2026 22:49:43 +0100 Subject: [PATCH 21/32] add san vlan --- hosts/physiques/cephiroth/networking.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 15c7d40..7c87116 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -10,6 +10,10 @@ id = 3; interface = "bond0"; }; + san = { + id = 4; + interface = "bond0"; + }; ceph = { id = 6; interface = "bond0"; From 053b0b0de5af5f3c65400884db2f3cc23809ef0d Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 14:42:51 +0100 Subject: [PATCH 22/32] add public and cluster network --- hosts/physiques/cephiroth/ceph.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 0cbf089..484142a 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -48,6 +48,8 @@ fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0"; monHost = "172.16.6.3"; monInitialMembers = "cephiroth"; + publicNetwork = "172.16.4.0/8"; + clusterNetwork = "172.16.6.0/8"; }; client = { From c22ba21761d6029adab34f000a9bb3f9219ff7b7 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 14:46:41 +0100 Subject: [PATCH 23/32] test config host --- hosts/physiques/cephiroth/ceph.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 484142a..ff85701 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -19,7 +19,7 @@ mon = { enable = true; extraConfig = { - "mon host" = "172.16.6.3"; + "mon host" = "172.16.4.3"; "mon initial members" = "cephiroth"; }; daemons = ["cephiroth"]; From 12f22d063ec3d8622d54d8357057e91e3021cd4f Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 14:48:28 +0100 Subject: [PATCH 24/32] another test --- hosts/physiques/cephiroth/ceph.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index ff85701..e681403 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -46,7 +46,7 @@ clusterName = "ceph"; # fsid unique par node, généré avec `uuidgen` fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0"; - monHost = "172.16.6.3"; + monHost = "172.16.4.3"; monInitialMembers = "cephiroth"; publicNetwork = "172.16.4.0/8"; clusterNetwork = "172.16.6.0/8"; From 3b38c9ab0b4c107f1e3abd448c42d368621cd7f8 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 14:54:40 +0100 Subject: [PATCH 25/32] remove firewall ceph --- hosts/physiques/cephiroth/networking.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 7c87116..93b91b2 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -4,6 +4,7 @@ networking = { dhcpcd.enable = false; + firewall = false; vlans = { srv-nat = { From 6e1cd9b6d78d9ff36c367f985f61c1a3989564b7 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 14:56:54 +0100 Subject: [PATCH 26/32] fix: correct parameter firewall --- hosts/physiques/cephiroth/networking.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 93b91b2..1dd7ea6 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -4,7 +4,7 @@ networking = { dhcpcd.enable = false; - firewall = false; + firewall.enable = false; vlans = { srv-nat = { From 3842a07f9326052113823ffb2be1ae067bd8177a Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 15:16:27 +0100 Subject: [PATCH 27/32] fix: addr mask --- hosts/physiques/cephiroth/ceph.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index e681403..888cd26 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -13,6 +13,10 @@ # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine # les machines physique ont des id < 100, donc ca fonctionne daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"]; + extraConfig = { + "public_addr" = "172.16.4.3"; + "cluster_addr" = "172.16.6.3"; + }; }; # monitor, garde la map du cluster @@ -48,8 +52,8 @@ fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0"; monHost = "172.16.4.3"; monInitialMembers = "cephiroth"; - publicNetwork = "172.16.4.0/8"; - clusterNetwork = "172.16.6.0/8"; + publicNetwork = "172.16.4.0/24"; + clusterNetwork = "172.16.6.0/24"; }; client = { From 5d8d0be570e415b2111df2cec37a0ceac94f241c Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 15:59:03 +0100 Subject: [PATCH 28/32] add another monitor to switch ip addr --- hosts/physiques/cephiroth/ceph.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 888cd26..7e13eba 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -13,10 +13,6 @@ # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine # les machines physique ont des id < 100, donc ca fonctionne daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"]; - extraConfig = { - "public_addr" = "172.16.4.3"; - "cluster_addr" = "172.16.6.3"; - }; }; # monitor, garde la map du cluster @@ -26,7 +22,7 @@ "mon host" = "172.16.4.3"; "mon initial members" = "cephiroth"; }; - daemons = ["cephiroth"]; + daemons = ["cephiroth" "cephiroth-san"]; }; # Dashboard From ed5aac805d4f0a81685b700abb29d027dc6a7be3 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 16:58:59 +0100 Subject: [PATCH 29/32] modify mtu --- hosts/physiques/cephiroth/networking.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index 1dd7ea6..beb112a 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -45,6 +45,7 @@ }; }; ceph = { + mtu = 9000; ipv4 = { addresses = [ { @@ -55,6 +56,7 @@ }; }; adm = { + mtu = 9000; ipv4 = { addresses = [ { @@ -65,6 +67,7 @@ }; }; san = { + mtu = 9000; ipv4 = { addresses = [ { From 78274fd0fa4f680ef79052d01d76b4c305132ae3 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Sun, 1 Feb 2026 22:56:04 +0100 Subject: [PATCH 30/32] fix: add public and cluster ip --- hosts/physiques/cephiroth/ceph.nix | 10 ++++++++-- hosts/physiques/cephiroth/networking.nix | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hosts/physiques/cephiroth/ceph.nix b/hosts/physiques/cephiroth/ceph.nix index 7e13eba..8d72a15 100644 --- a/hosts/physiques/cephiroth/ceph.nix +++ b/hosts/physiques/cephiroth/ceph.nix @@ -13,6 +13,10 @@ # je propose comme convention, entier suivi de l'id à 2 chiffre de la machine # les machines physique ont des id < 100, donc ca fonctionne daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"]; + extraConfig = { + "public_addr" = "172.16.4.3"; + "cluster_addr" = "172.16.6.3"; + }; }; # monitor, garde la map du cluster @@ -22,7 +26,7 @@ "mon host" = "172.16.4.3"; "mon initial members" = "cephiroth"; }; - daemons = ["cephiroth" "cephiroth-san"]; + daemons = ["cephiroth"]; }; # Dashboard @@ -30,7 +34,9 @@ enable = true; daemons = ["cephiroth"]; extraConfig = { - "mgr_initial_modules" = "dashboard"; + "mgr_initial_modules" = "dashboard prometheus"; + # TODO: setup prometheus listen addr + # le port de prometheus est 9283 pour ceph }; }; diff --git a/hosts/physiques/cephiroth/networking.nix b/hosts/physiques/cephiroth/networking.nix index beb112a..027488a 100644 --- a/hosts/physiques/cephiroth/networking.nix +++ b/hosts/physiques/cephiroth/networking.nix @@ -33,6 +33,8 @@ interfaces = { ens2f0.mtu = 9000; ens2f1.mtu = 9000; + + bonds.mtu = 9000; srv-nat = { ipv4 = { From f65fc106c76bf48340dc4522893d5ae632fad843 Mon Sep 17 00:00:00 2001 From: Lzebulon Date: Fri, 13 Feb 2026 22:13:00 +0100 Subject: [PATCH 31/32] add information probleme possible --- hosts/physiques/cephiroth/ceph.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/hosts/physiques/cephiroth/ceph.md b/hosts/physiques/cephiroth/ceph.md index 60076cc..b04a12f 100644 --- a/hosts/physiques/cephiroth/ceph.md +++ b/hosts/physiques/cephiroth/ceph.md @@ -77,6 +77,22 @@ Pour créer un osd (attention de prendre le bon disque parce que ca supprime les sudo ceph-volume lvm create --data /dev/sdf --no-systemd ``` +# Problèmes + +Si avec `ceph -s` vous avez : + +`1 monitors have not enabled msgr2`, +activez le msgr2, afin que les vms proxmox puissent utiliser les disques : +```bash +sudo ceph mon enable-msgr2 +``` +`mons are allowing insecure global_id reclaim` +alors +```bash +ceph config set mon auth_allow_insecure_global_id_reclaim false +``` + + # Info - Monitor : c'est lui qui garde la map d'ou est quoi @@ -114,3 +130,14 @@ gourmand en ressources. - migration nfs -> cephfs ? # Plan Migration + + + +# Monitoring + +Les dashboard grafana viennent de : + +Pour activer prometheus (port 9283 par defaut) : +```bash +ceph mgr module enable prometheus +``` From df4c48707bdf505a24acfab6943080cd5e2e6c9f Mon Sep 17 00:00:00 2001 From: Pyjacpp Date: Sat, 21 Nov 2026 15:05:46 +0100 Subject: [PATCH 32/32] Fix cephiroth/info.nix --- hosts/physiques/cephiroth/default.nix | 18 ++---------------- hosts/physiques/cephiroth/info.nix | 12 +++++++----- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/hosts/physiques/cephiroth/default.nix b/hosts/physiques/cephiroth/default.nix index f4c727a..b680334 100644 --- a/hosts/physiques/cephiroth/default.nix +++ b/hosts/physiques/cephiroth/default.nix @@ -25,7 +25,7 @@ # path = "/boot-fallback"; # } { - devices = ["nodev"]; + devices = [ "nodev" ]; path = "/boot-fallback"; } ]; @@ -37,22 +37,8 @@ }; boot.supportedFilesystems = [ "zfs" ]; - boot.zfs.extraPools = ["head"]; + boot.zfs.extraPools = [ "head" ]; boot.loader.efi.canTouchEfiVariables = true; - - crans = { - enable = true; - - networking = { - id = 3; - # On doit utiliser des vlans et du bond, donc - # on desactive la config network des vms - enable = lib.mkForce false; - }; - resticClient.enable = false; - }; - - system.stateVersion = "24.05"; } diff --git a/hosts/physiques/cephiroth/info.nix b/hosts/physiques/cephiroth/info.nix index e2f7b28..1da82c6 100644 --- a/hosts/physiques/cephiroth/info.nix +++ b/hosts/physiques/cephiroth/info.nix @@ -1,14 +1,14 @@ -{ +{ lib, ... }: { name = "cephiroth"; description = "Serveur de stockage ceph"; id = 3; - # Sous Nix mais dans une autre branche - isDebian = true; + enable = true; networking = { - adm.enable = true; - san.enable = true; + # On doit utiliser des vlans et du bond, donc + # on desactive la config network des vms + enable = lib.mkForce false; switch = { carapuce = 12; @@ -16,6 +16,8 @@ }; }; + resticClient.enable = false; + ilo = { id = 53; switch.salameche = 20;