ansible-fet/doc/ariane.md

ariane

Hard disks

	V1	V2	V3	V4
H1	00 /dev/sda 6TB WD-WX21D36PP32E	04	08	12
H2	01 /dev/sdb 6TB WD-WX21D36PPLPH	05	09	13
H3	02 /dev/sdc 6TB WD-WX21D36PP0K1	06	10	14
H4	03 /dev/sdd 6TB WD-WXB1HB4MJCMM	07	11	15

Debian setup install steps

Boot Image Debian Netinstall on USB stick, Advanced Options->Expert Install
Language: English, Location: other->Europe->Austria
Locale: en_US.UTF-8, Additional Locale: de_AT.UTF-8, System Locale: en_US.UTF-8
Keyboard: German
Detect and mount CD-ROM, Load installer components: no extra
auf KISTL statische IP 192.168.86.7/24 für ariane.fet.htu.tuwien.ac.at einrichten
Detect network hardware, Configure Network: enp8s0: Hostname: ariane, IP 192.168.86.7/24 domain fet.htu.tuwien.ac.at
Setup Users and Passwords: shadow, user petra
Configure the clock: NTP Server: tutimea.tuwien.ac.at
Detect disks, partition disks: manual
RAID1 md0 with both SSDs https://blog.sleeplessbeastie.eu/2013/10/04/how-to-configure-software-raid1-during-installation-process/
/dev/nvmeXn1p1 32GB primary Volume für / in btrfs 32GB
/dev/nvmeXn1p2 80 GB primary Volume
ZFS RAIDZ-2 on is already on 4 x 6TB disks = 12TB 0 spares
partition1 ariane-root btrfs noatime, discard
Generic Kernel
Mirror: gd.tuwien.ac.at, allow backported
install with ssh server and standard sys utilities
Install Grub on /dev/nvme0n1 (to removable media path)
Install Grub on /dev/nvme1n1
reboot, log in as root
edit /etc/ssh/sshd_config set PermitRootlogin to yes
service sshd reload
ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub
ssh-copy-id root@ariane from client
edit /etc/ssh/sshd_config set PermitRootlogin to Prohibit-Password

Install ZFS

apt-get install zfsutils-linux
modprobe zfs
systemctl restart zfs-import-cache
systemctl restart zfs-import-scan
systemctl restart zfs-mount
systemctl restart zfs-share

Initial ZFS Setup

Find disk serial numbers

for i in a b c d e f g h i; do echo -n "/dev/sd$i: "; hdparm -I /dev/sd$i | awk '/Serial Number/ {print $3}'; done
nvme list

Wipe disks

mdadm --zero-superblock --force /dev/sda
sgdisk --zap-all /dev/sdX

Partition disks

sgdisk -n1:0:0 -t1:BF01 /dev/sdX

Format LUKS containers (and set passphrase)

cryptsetup luksFormat /dev/disk/by-id/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PP32E-part1
cryptsetup luksFormat /dev/disk/by-id/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PPLPH-part1
cryptsetup luksFormat /dev/disk/by-id/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PP0K1-part1
cryptsetup luksFormat /dev/disk/by-id/ata-WDC_WD60EFRX-68L0BN1_WD-WXB1HB4MJCMM-part1

Create zpools

zfs_mount.sh mount
zpool create -o ashift=12 -o autoexpand=on -o autoreplace=on -O atime=off -O compression=lz4 \
-O acltype=posixacl -O xattr=sa zv1 raidz2 \
/dev/mapper/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PP32E-part1 \
/dev/mapper/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PPLPH-part1 \
/dev/mapper/ata-WDC_WD60EFRX-68L0BN1_WD-WX21D36PP0K1-part1 \
/dev/mapper/ata-WDC_WD60EFRX-68L0BN1_WD-WXB1HB4MJCMM-part1
zpool create -o ashift=12 -O atime=off -O compression=lz4 ssd mirror /dev/disk/by-id/nvme-INTEL_SSDPEKKW128G7_BTPY63830DUM128A-part2 /dev/disk/by-id/nvme-INTEL_SSDPEKKW128G7_BTPY63750L9G128A-part2

Create datasets

zfs create zv1/homes
zfs create zv1/daten
zfs create zv1/fotos
chown root:nogroup /zv1/homes
chown root:2000 /zv1/daten/ /zv1/fotos/
zfs create -o canmount=off -o setuid=off -o exec=off ssd/var
zfs create -o com.sun:auto-snapshot=false -o mountpoint=/var/lib/nfs ssd/var/nfs
zfs create -o com.sun:auto-snapshot=false -o exec=on ssd/var/cache
zfs create ssd/var/log
zfs create -o setuid=on -o exec=on ssd/var/lxc
mv /var/cache/* /ssd/var/cache/
zfs set mountpoint=/var/cache/ ssd/var/cache
mv /var/log/* /ssd/var/log/
zfs set mountpoint=/var/log ssd/var/log
mv /var/lib/lxc/* /ssd/var/lxc/
zfs set mountpoint=/var/lib/lxc ssd/var/lxc
zfs create -o com.sun:auto-snapshot=false zv1/sojus
zfs create -o com.sun:auto-snapshot=false zv1/daten/Scans
zfs create -o setuid=off -o exec=off zv1/zyklon
chown 997:996 /zv1/zyklon

Set dataset quota

zfs set quota=1T zv1/homes zv1/daten zv1/fotos
zfs set quota=3T zv1/sojus
zfs set quota=5G zv1/daten/Scans

If intend using ACL someday

zfs set xattr=sa zv1
zfs set acltype=posixacl zv1
zfs set aclinherit=passthrough zv1

If ACL for ZFS NFS share is implemented someday (not done!)

setfacl -R --set u::rwX,g::rwX,o::rwX /zv1/{daten,fotos}/
setfacl -d -R --set u::rwX,g::rwX,o::rwX /zv1/{daten,fotos}/

Transfer data

# https://github.com/rubo77/rsync-homedir-excludes
rsync --stats -avhP --exclude-from=/var/tmp/ignorelist root@atlas:/home/ /zv1/homes/
rsync --stats -avhP --exclude Pictures root@atlas:/mnt/save/fet_daten/ /zv1/daten/
rsync --stats -avhP root@atlas:/mnt/save/fet_daten/Pictures/ /zv1/fotos/

Share via NFS

zfs set sharenfs="rw=@192.168.86.0/24,insecure,sync,no_subtree_check,root_squash,no_wdelay" zv1/homes
zfs set sharenfs="rw=@192.168.86.0/24,insecure,sync,no_subtree_check,root_squash,crossmnt" zv1/daten
zfs set sharenfs="rw=@192.168.86.0/24,insecure,sync,no_subtree_check,root_squash,crossmnt" zv1/fotos
showmount -e

Before reinstalling

zfs export

After reinstalling

./ansible-playbook -i hosts/production site.yml --tags common --limit ariane
ssh ariane apt-get update && apt-get upgrade && reboot
./ansible-playbook -i hosts/production site.yml --tags zfs --limit ariane
zfs import
zpool import zv1
zfs mount -a

ZFS Maintenance

zpool status -v
zpool list -v
zpool get all
zpool history
zpool scrub zv1
zfs list -o space

show and delete all snapshots

zfs list -t snapshot
zfs list -H -o name -t snapshot | xargs -n1 zfs destroy

MDADM Maintenance

mdadm --misc --detail /dev/md0
echo check > /sys/block/md0/md/sync_action
watch -n 0.1 cat /proc/mdstat

Start System

zfs_mount.sh mount # Enter Passphrase
# start all LXC containers

Reboot System

# stop all LXC containers
zfs_mount.sh reboot

Shutdown System

# stop all LXC containers
zfs_mount.sh unmount
halt -p