mirror of https://gitlab.crans.org/nounous/nixos
Merge branch 'cephiroth-v3' into 'main'
Draft: Ceph See merge request nounous/nixos!108merge-requests/108/merge
commit
ed49c52d1c
|
|
@ -0,0 +1,143 @@
|
|||
# Ceph
|
||||
|
||||
Ce fichier explique les tests et etapes effectue avec ceph.
|
||||
|
||||
|
||||
|
||||
+ Déployer une conf minimal avec ceph et `services.ceph.global` deploye.
|
||||
+ Creer une clé pour le cluster
|
||||
```bash
|
||||
sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
|
||||
```
|
||||
+ Creer un client admin
|
||||
```bash
|
||||
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
|
||||
```
|
||||
+ Creer un bootstrap osd
|
||||
```bash
|
||||
sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
|
||||
```
|
||||
+ On ajoute les cle generer au keyring
|
||||
```bash
|
||||
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
|
||||
sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
```
|
||||
(ca copie juste les deux clé précédente dans `/tmp/ceph.mon.keyring`)
|
||||
+ on donne le keyring a l'user ceph
|
||||
```bash
|
||||
sudo chown ceph:ceph /tmp/ceph.mon.keyring
|
||||
```
|
||||
+ Créer une monitor map
|
||||
```bash
|
||||
monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
|
||||
```
|
||||
+ Créer un dossier pour le monitor
|
||||
```bash
|
||||
sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
|
||||
```
|
||||
+ On ajoute la monitor map et le keyring au monitor
|
||||
```bash
|
||||
sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
|
||||
```
|
||||
|
||||
On check si le service monitor start bien
|
||||
|
||||
Pour voir le status du cluster
|
||||
```bash
|
||||
sudo ceph -s
|
||||
```
|
||||
|
||||
|
||||
# Dashboard
|
||||
- check si le module dashboard est activé
|
||||
```
|
||||
sudo ceph mgr module ls
|
||||
```
|
||||
- pour activer si pas activer
|
||||
```bash
|
||||
ceph mgr module enable dashboard
|
||||
```
|
||||
|
||||
Dans un premier temps pour le test j'ai desactiver le ssl :
|
||||
```bash
|
||||
ceph config set mgr mgr/dashboard/ssl false
|
||||
```
|
||||
|
||||
Pour voir sur quoi le dashboard ecoute (apres avoir desactive ssl)
|
||||
```bash
|
||||
sudo ceph mgr services
|
||||
```
|
||||
|
||||
Pour les user check https://docs.ceph.com/en/latest/mgr/dashboard/#username-and-password
|
||||
|
||||
# OSD
|
||||
|
||||
Pour créer un osd (attention de prendre le bon disque parce que ca supprime les données du disque) :
|
||||
```bash
|
||||
sudo ceph-volume lvm create --data /dev/sdf --no-systemd
|
||||
```
|
||||
|
||||
# Problèmes
|
||||
|
||||
Si avec `ceph -s` vous avez :
|
||||
|
||||
`1 monitors have not enabled msgr2`,
|
||||
activez le msgr2, afin que les vms proxmox puissent utiliser les disques :
|
||||
```bash
|
||||
sudo ceph mon enable-msgr2
|
||||
```
|
||||
`mons are allowing insecure global_id reclaim`
|
||||
alors
|
||||
```bash
|
||||
ceph config set mon auth_allow_insecure_global_id_reclaim false
|
||||
```
|
||||
|
||||
|
||||
# Info
|
||||
|
||||
- Monitor : c'est lui qui garde la map d'ou est quoi
|
||||
- Manager : il check si tout va bien, c'est lui qui a le dashboard
|
||||
- OSD : ca gere 1 disque physique
|
||||
|
||||
|
||||
# Proposition Architecture
|
||||
|
||||
## CephFS
|
||||
Il semble qu'il est préférable d'avoir 1 cephfs avec des sous répertoire.
|
||||
La raison principale est qu'il faut 1 mds/cephfs, et ca semble relativement
|
||||
gourmand en ressources.
|
||||
|
||||
- cephfs
|
||||
- home-nounou
|
||||
- home-adh
|
||||
- mail
|
||||
- mirror
|
||||
|
||||
> Note: on pourrait penser bouger le mirror vers du S3 ?
|
||||
|
||||
## RBD (raw block devices):
|
||||
- vm-adh
|
||||
- vm-nounou
|
||||
|
||||
|
||||
# Question en suspend
|
||||
- backup ?
|
||||
- postgresql
|
||||
- monitoring
|
||||
- il y a du prometheus et des dashboard grafana ca devrait le faire
|
||||
- user :
|
||||
- 1 user/"volume" ou 1user/host, autres ?
|
||||
- migration nfs -> cephfs ?
|
||||
|
||||
# Plan Migration
|
||||
|
||||
|
||||
|
||||
# Monitoring
|
||||
|
||||
Les dashboard grafana viennent de : <https://github.com/ceph/ceph/tree/main/monitoring/ceph-mixin/dashboards_out>
|
||||
|
||||
Pour activer prometheus (port 9283 par defaut) :
|
||||
```bash
|
||||
ceph mgr module enable prometheus
|
||||
```
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
{ pkgs, ... }:
|
||||
|
||||
{
|
||||
# La doc ceph : https://docs.ceph.com/en/quincy/install/manual-deployment/
|
||||
|
||||
services.ceph = {
|
||||
enable = true;
|
||||
|
||||
# osd, 1 deamon/disk avec un id unique dans le cluster.
|
||||
osd = {
|
||||
enable = true;
|
||||
|
||||
# je propose comme convention, entier suivi de l'id à 2 chiffre de la machine
|
||||
# les machines physique ont des id < 100, donc ca fonctionne
|
||||
daemons = ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11"];
|
||||
extraConfig = {
|
||||
"public_addr" = "172.16.4.3";
|
||||
"cluster_addr" = "172.16.6.3";
|
||||
};
|
||||
};
|
||||
|
||||
# monitor, garde la map du cluster
|
||||
mon = {
|
||||
enable = true;
|
||||
extraConfig = {
|
||||
"mon host" = "172.16.4.3";
|
||||
"mon initial members" = "cephiroth";
|
||||
};
|
||||
daemons = ["cephiroth"];
|
||||
};
|
||||
|
||||
# Dashboard
|
||||
mgr = {
|
||||
enable = true;
|
||||
daemons = ["cephiroth"];
|
||||
extraConfig = {
|
||||
"mgr_initial_modules" = "dashboard prometheus";
|
||||
# TODO: setup prometheus listen addr
|
||||
# le port de prometheus est 9283 pour ceph
|
||||
};
|
||||
};
|
||||
|
||||
# pour cephFS
|
||||
mds = {
|
||||
enable = true;
|
||||
daemons = ["cephiroth"];
|
||||
};
|
||||
|
||||
global = {
|
||||
# Comme nom de cluster on choisit `ceph` (la valeur par défaut),
|
||||
# car on aura qu'un cluster.
|
||||
clusterName = "ceph";
|
||||
# fsid unique par node, généré avec `uuidgen`
|
||||
fsid = "af0a334a-3356-4fb6-800c-95b4a99232c0";
|
||||
monHost = "172.16.4.3";
|
||||
monInitialMembers = "cephiroth";
|
||||
publicNetwork = "172.16.4.0/24";
|
||||
clusterNetwork = "172.16.6.0/24";
|
||||
};
|
||||
|
||||
client = {
|
||||
enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
# automaticaly start osd when we restart the server
|
||||
systemd.services.ceph-mesh = {
|
||||
enable = true;
|
||||
description = "Ceph OSD Bindings";
|
||||
unitConfig = {
|
||||
After = "local-fs.target";
|
||||
Wants = "local-fs.target";
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
KillMode = "none";
|
||||
Environment = "CEPH_VOLUME_TIMEOUT=10000 PATH=$PATH:/run/current-system/sw/bin/";
|
||||
ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT /run/current-system/sw/bin/ceph-volume lvm activate --all --no-systemd'";
|
||||
TimeoutSec = 0;
|
||||
};
|
||||
wantedBy = ["multi-user.target"];
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.ceph pkgs.smartmontools ];
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
{ pkgs, lib, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
./networking.nix
|
||||
./ceph.nix
|
||||
];
|
||||
|
||||
networking.hostId = "bbdd1133";
|
||||
networking.hostName = "cephiroth";
|
||||
|
||||
boot.loader.grub = {
|
||||
enable = true;
|
||||
efiSupport = true;
|
||||
zfsSupport = true;
|
||||
device = "nodev";
|
||||
mirroredBoots = [
|
||||
# {
|
||||
# devices = ["/dev/sdi"];
|
||||
# path = "/boot";
|
||||
# }
|
||||
# {
|
||||
# devices = [ "/dev/sdj" ];
|
||||
# path = "/boot-fallback";
|
||||
# }
|
||||
{
|
||||
devices = [ "nodev" ];
|
||||
path = "/boot-fallback";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
boot.tmp = {
|
||||
useTmpfs = true;
|
||||
cleanOnBoot = true;
|
||||
};
|
||||
|
||||
boot.supportedFilesystems = [ "zfs" ];
|
||||
boot.zfs.extraPools = [ "head" ];
|
||||
boot.loader.efi.canTouchEfiVariables = true;
|
||||
|
||||
system.stateVersion = "24.05";
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "uhci_hcd" "hpsa" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-intel" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "head/root";
|
||||
fsType = "zfs";
|
||||
options = ["zfsutil"];
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/C900-92D1";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0022" "dmask=0022" ];
|
||||
};
|
||||
|
||||
fileSystems."/boot-fallback" =
|
||||
{ device = "/dev/disk/by-uuid/C931-84A4";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0022" "dmask=0022" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
||||
# (the default) this is the recommended approach. When using systemd-networkd it's
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.bond0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.bond0.10.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.bond0.3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.bonding_masters.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno4.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno49.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno50.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno51.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.eno52.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp11s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp11s0f1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
{
|
||||
{ lib, ... }: {
|
||||
name = "cephiroth";
|
||||
description = "Serveur de stockage ceph";
|
||||
id = 3;
|
||||
|
||||
# Sous Nix mais dans une autre branche
|
||||
isDebian = true;
|
||||
enable = true;
|
||||
|
||||
networking = {
|
||||
adm.enable = true;
|
||||
san.enable = true;
|
||||
# On doit utiliser des vlans et du bond, donc
|
||||
# on desactive la config network des vms
|
||||
enable = lib.mkForce false;
|
||||
|
||||
switch = {
|
||||
carapuce = 12;
|
||||
|
|
@ -16,6 +16,8 @@
|
|||
};
|
||||
};
|
||||
|
||||
resticClient.enable = false;
|
||||
|
||||
ilo = {
|
||||
id = 53;
|
||||
switch.salameche = 20;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
{ ... }:
|
||||
|
||||
{
|
||||
networking = {
|
||||
|
||||
dhcpcd.enable = false;
|
||||
firewall.enable = false;
|
||||
|
||||
vlans = {
|
||||
srv-nat = {
|
||||
id = 3;
|
||||
interface = "bond0";
|
||||
};
|
||||
san = {
|
||||
id = 4;
|
||||
interface = "bond0";
|
||||
};
|
||||
ceph = {
|
||||
id = 6;
|
||||
interface = "bond0";
|
||||
};
|
||||
adm = {
|
||||
id = 10;
|
||||
interface = "bond0";
|
||||
};
|
||||
};
|
||||
|
||||
bonds.bond0 = {
|
||||
interfaces = [ "ens2f0" "ens2f1" ];
|
||||
driverOptions.mode = "802.3ad";
|
||||
};
|
||||
|
||||
interfaces = {
|
||||
ens2f0.mtu = 9000;
|
||||
ens2f1.mtu = 9000;
|
||||
|
||||
bonds.mtu = 9000;
|
||||
|
||||
srv-nat = {
|
||||
ipv4 = {
|
||||
addresses = [
|
||||
{
|
||||
address = "172.16.3.3";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
ceph = {
|
||||
mtu = 9000;
|
||||
ipv4 = {
|
||||
addresses = [
|
||||
{
|
||||
address = "172.16.6.3";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
adm = {
|
||||
mtu = 9000;
|
||||
ipv4 = {
|
||||
addresses = [
|
||||
{
|
||||
address = "172.16.10.3";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
san = {
|
||||
mtu = 9000;
|
||||
ipv4 = {
|
||||
addresses = [
|
||||
{
|
||||
address = "172.16.4.3";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
defaultGateway = "172.16.3.99";
|
||||
nameservers = [ "172.16.10.128" ];
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue