Merge branch 'cephiroth-v3' into 'main'

Draft: Ceph

See merge request nounous/nixos!108
merge-requests/108/merge
Pyjacpp 2026-11-21 15:11:31 +01:00
commit ed49c52d1c
6 changed files with 423 additions and 5 deletions

View File

@ -0,0 +1,143 @@
# Ceph
Ce fichier explique les tests et etapes effectue avec ceph.
+ Déployer une conf minimal avec ceph et `services.ceph.global` deploye.
+ Creer une clé pour le cluster
```bash
sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
```
+ Creer un client admin
```bash
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
```
+ Creer un bootstrap osd
```bash
sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
```
+ On ajoute les cle generer au keyring
```bash
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
```
(ca copie juste les deux clé précédente dans `/tmp/ceph.mon.keyring`)
+ on donne le keyring a l'user ceph
```bash
sudo chown ceph:ceph /tmp/ceph.mon.keyring
```
+ Créer une monitor map
```bash
monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
```
+ Créer un dossier pour le monitor
```bash
sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
```
+ On ajoute la monitor map et le keyring au monitor
```bash
sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
```
On check si le service monitor start bien
Pour voir le status du cluster
```bash
sudo ceph -s
```
# Dashboard
- check si le module dashboard est activé
```
sudo ceph mgr module ls
```
- pour activer si pas activer
```bash
ceph mgr module enable dashboard
```
Dans un premier temps pour le test j'ai desactiver le ssl :
```bash
ceph config set mgr mgr/dashboard/ssl false
```
Pour voir sur quoi le dashboard ecoute (apres avoir desactive ssl)
```bash
sudo ceph mgr services
```
Pour les user check https://docs.ceph.com/en/latest/mgr/dashboard/#username-and-password
# OSD
Pour créer un osd (attention de prendre le bon disque parce que ca supprime les données du disque) :
```bash
sudo ceph-volume lvm create --data /dev/sdf --no-systemd
```
# Problèmes
Si avec `ceph -s` vous avez :
`1 monitors have not enabled msgr2`,
activez le msgr2, afin que les vms proxmox puissent utiliser les disques :
```bash
sudo ceph mon enable-msgr2
```
`mons are allowing insecure global_id reclaim`
alors
```bash
ceph config set mon auth_allow_insecure_global_id_reclaim false
```
# Info
- Monitor : c'est lui qui garde la map d'ou est quoi
- Manager : il check si tout va bien, c'est lui qui a le dashboard
- OSD : ca gere 1 disque physique
# Proposition Architecture
## CephFS
Il semble qu'il est préférable d'avoir 1 cephfs avec des sous répertoire.
La raison principale est qu'il faut 1 mds/cephfs, et ca semble relativement
gourmand en ressources.
- cephfs
- home-nounou
- home-adh
- mail
- mirror
> Note: on pourrait penser bouger le mirror vers du S3 ?
## RBD (raw block devices):
- vm-adh
- vm-nounou
# Question en suspend
- backup ?
- postgresql
- monitoring
- il y a du prometheus et des dashboard grafana ca devrait le faire
- user :
- 1 user/"volume" ou 1user/host, autres ?
- migration nfs -> cephfs ?
# Plan Migration
# Monitoring
Les dashboard grafana viennent de : <https://github.com/ceph/ceph/tree/main/monitoring/ceph-mixin/dashboards_out>
Pour activer prometheus (port 9283 par defaut) :
```bash
ceph mgr module enable prometheus
```

View File

@ -0,0 +1,85 @@
{ pkgs, ... }:
{
# La doc ceph : https://docs.ceph.com/en/quincy/install/manual-deployment/
services.ceph = {
enable = true;
# osd, 1 deamon/disk avec un id unique dans le cluster.
osd = {
enable = true;
# je propose comme convention, entier suivi de l'id à 2 chiffre de la machine
# les machines physique ont des id < 100, donc ca fonctionne
daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"];
extraConfig = {
"public_addr" = "172.16.4.3";
"cluster_addr" = "172.16.6.3";
};
};
# monitor, garde la map du cluster
mon = {
enable = true;
extraConfig = {
"mon host" = "172.16.4.3";
"mon initial members" = "cephiroth";
};
daemons = ["cephiroth"];
};
# Dashboard
mgr = {
enable = true;
daemons = ["cephiroth"];
extraConfig = {
"mgr_initial_modules" = "dashboard prometheus";
# TODO: setup prometheus listen addr
# le port de prometheus est 9283 pour ceph
};
};
# pour cephFS
mds = {
enable = true;
daemons = ["cephiroth"];
};
global = {
# Comme nom de cluster on choisit `ceph` (la valeur par défaut),
# car on aura qu'un cluster.
clusterName = "ceph";
# fsid unique par node, généré avec `uuidgen`
fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0";
monHost = "172.16.4.3";
monInitialMembers = "cephiroth";
publicNetwork = "172.16.4.0/24";
clusterNetwork = "172.16.6.0/24";
};
client = {
enable = true;
};
};
# automaticaly start osd when we restart the server
systemd.services.ceph-mesh = {
enable = true;
description = "Ceph OSD Bindings";
unitConfig = {
After = "local-fs.target";
Wants = "local-fs.target";
};
serviceConfig = {
Type = "oneshot";
KillMode = "none";
Environment = "CEPH_VOLUME_TIMEOUT=10000 PATH=$PATH:/run/current-system/sw/bin/";
ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT /run/current-system/sw/bin/ceph-volume lvm activate --all --no-systemd'";
TimeoutSec = 0;
};
wantedBy = ["multi-user.target"];
};
environment.systemPackages = [ pkgs.ceph pkgs.smartmontools ];
}

View File

@ -0,0 +1,44 @@
{ pkgs, lib, ... }:
{
imports = [
./hardware-configuration.nix
./networking.nix
./ceph.nix
];
networking.hostId = "bbdd1133";
networking.hostName = "cephiroth";
boot.loader.grub = {
enable = true;
efiSupport = true;
zfsSupport = true;
device = "nodev";
mirroredBoots = [
# {
# devices = ["/dev/sdi"];
# path = "/boot";
# }
# {
# devices = [ "/dev/sdj" ];
# path = "/boot-fallback";
# }
{
devices = [ "nodev" ];
path = "/boot-fallback";
}
];
};
boot.tmp = {
useTmpfs = true;
cleanOnBoot = true;
};
boot.supportedFilesystems = [ "zfs" ];
boot.zfs.extraPools = [ "head" ];
boot.loader.efi.canTouchEfiVariables = true;
system.stateVersion = "24.05";
}

View File

@ -0,0 +1,58 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "uhci_hcd" "hpsa" "usbhid" "usb_storage" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-intel" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "head/root";
fsType = "zfs";
options = ["zfsutil"];
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/C900-92D1";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
fileSystems."/boot-fallback" =
{ device = "/dev/disk/by-uuid/C931-84A4";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.bond0.useDHCP = lib.mkDefault true;
# networking.interfaces.bond0.10.useDHCP = lib.mkDefault true;
# networking.interfaces.bond0.3.useDHCP = lib.mkDefault true;
# networking.interfaces.bonding_masters.useDHCP = lib.mkDefault true;
# networking.interfaces.eno1.useDHCP = lib.mkDefault true;
# networking.interfaces.eno2.useDHCP = lib.mkDefault true;
# networking.interfaces.eno3.useDHCP = lib.mkDefault true;
# networking.interfaces.eno4.useDHCP = lib.mkDefault true;
# networking.interfaces.eno49.useDHCP = lib.mkDefault true;
# networking.interfaces.eno50.useDHCP = lib.mkDefault true;
# networking.interfaces.eno51.useDHCP = lib.mkDefault true;
# networking.interfaces.eno52.useDHCP = lib.mkDefault true;
# networking.interfaces.enp11s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp11s0f1.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

View File

@ -1,14 +1,14 @@
{ { lib, ... }: {
name = "cephiroth"; name = "cephiroth";
description = "Serveur de stockage ceph"; description = "Serveur de stockage ceph";
id = 3; id = 3;
# Sous Nix mais dans une autre branche enable = true;
isDebian = true;
networking = { networking = {
adm.enable = true; # On doit utiliser des vlans et du bond, donc
san.enable = true; # on desactive la config network des vms
enable = lib.mkForce false;
switch = { switch = {
carapuce = 12; carapuce = 12;
@ -16,6 +16,8 @@
}; };
}; };
resticClient.enable = false;
ilo = { ilo = {
id = 53; id = 53;
switch.salameche = 20; switch.salameche = 20;

View File

@ -0,0 +1,86 @@
{ ... }:
{
networking = {
dhcpcd.enable = false;
firewall.enable = false;
vlans = {
srv-nat = {
id = 3;
interface = "bond0";
};
san = {
id = 4;
interface = "bond0";
};
ceph = {
id = 6;
interface = "bond0";
};
adm = {
id = 10;
interface = "bond0";
};
};
bonds.bond0 = {
interfaces = [ "ens2f0" "ens2f1" ];
driverOptions.mode = "802.3ad";
};
interfaces = {
ens2f0.mtu = 9000;
ens2f1.mtu = 9000;
bonds.mtu = 9000;
srv-nat = {
ipv4 = {
addresses = [
{
address = "172.16.3.3";
prefixLength = 24;
}
];
};
};
ceph = {
mtu = 9000;
ipv4 = {
addresses = [
{
address = "172.16.6.3";
prefixLength = 24;
}
];
};
};
adm = {
mtu = 9000;
ipv4 = {
addresses = [
{
address = "172.16.10.3";
prefixLength = 24;
}
];
};
};
san = {
mtu = 9000;
ipv4 = {
addresses = [
{
address = "172.16.4.3";
prefixLength = 24;
}
];
};
};
};
defaultGateway = "172.16.3.99";
nameservers = [ "172.16.10.128" ];
};
}