# Commands

Linux, Windows Server, NixOS, storage, network, Proxmox, ESXi, Hyper-V, Docker and Kubernetes. Source: https://merox.dev/commands

Anything in <angle-brackets> is a placeholder.

## Triage

### First minute on a sick box

```sh
# Uptime and load averages
uptime

# Kernel errors and warnings
dmesg -T --level=err,warn | tail -50

# Failed systemd units
systemctl --failed

# Errors logged since boot
journalctl -p err -b

# Pressure stall: is CPU, memory or I/O actually contended
grep . /proc/pressure/*

# Run queue, swapping, CPU split
vmstat 1 5

# Disk latency and saturation
iostat -xz 1 3

# Network throughput per interface
sar -n DEV 1 5

# TCP retransmits per second
sar -n TCP,ETCP 1 5

# Load per CPU core
mpstat -P ALL 1 3

# CPU and disk I/O per process
pidstat -u -d 1 3

# Top memory consumers
ps aux --sort=-%mem | head

# Top CPU consumers
ps aux --sort=-%cpu | head

# Processes stuck in D state (hung on I/O or NFS)
ps -eo stat,pid,wchan:32,cmd | awk '$1 ~ /^D/'

# Dump stacks of blocked tasks to the kernel log
echo w > /proc/sysrq-trigger && dmesg -T | tail -100

# OOM killer activity
journalctl -k -g 'killed process'

# Machine check and hardware errors
journalctl -k -g 'mce|machine check|hardware error'

# Recent reboots and shutdowns
last -x reboot shutdown | head

# Load earlier today, from sysstat history (Debian path)
sar -q -f /var/log/sysstat/sa<day-of-month>
```

### Memory

```sh
# Memory and swap
free -h

# Available memory, dirty pages, kernel slab
grep -E 'MemAvailable|Dirty|Slab|SUnreclaim' /proc/meminfo

# Kernel caches by size (memory gone but no process owns it)
slabtop -o -s c | head -20

# Processes using the most swap
grep VmSwap /proc/[0-9]*/status | sort -k2 -n | tail
```

### Limits that fail quietly

```sh
# System-wide file handles: used, free, max
cat /proc/sys/fs/file-nr

# Open files of a process against its limit
ls /proc/<pid>/fd | wc -l && grep 'open files' /proc/<pid>/limits

# Conntrack table full (drops new connections)
cat /proc/sys/net/netfilter/nf_conntrack_count /proc/sys/net/netfilter/nf_conntrack_max

# Threads in use against the PID limit
ps -eLf | wc -l && cat /proc/sys/kernel/pid_max

# Clock offset (breaks TLS, Kerberos, etcd)
chronyc tracking

# Time, timezone and NTP sync state
timedatectl
```

### Out of space

```sh
# Usage per real filesystem
df -hT -x tmpfs -x devtmpfs -x overlay

# Inodes exhausted (df shows space, writes still fail)
df -i

# Largest directories on one filesystem
du -xh --max-depth=1 / | sort -h | tail -20

# Interactive disk usage browser
ncdu -x /

# Files over 1 GB
find / -xdev -type f -size +1G -exec ls -lh {} +

# Deleted files still held open
lsof +L1

# Free a deleted-but-open file without restarting its process
: > /proc/<pid>/fd/<fd>

# Empty a live log without breaking the writer
truncate -s 0 <file>

# df and du disagree: look under the mountpoints
mkdir -p /mnt/rootfs && mount --bind / /mnt/rootfs && du -xh --max-depth=1 /mnt/rootfs | sort -h

# ext4: shrink the 5% root reserve on a data disk
tune2fs -m 1 /dev/<device>

# Shrink the systemd journal
journalctl --vacuum-size=500M
```

### Windows: first look

```powershell
# Last boot time
(Get-CimInstance Win32_OperatingSystem).LastBootUpTime

# Why it rebooted: 41 unexpected, 1074 initiated, 6008 dirty shutdown
Get-WinEvent -FilterHashtable @{LogName='System'; Id=41,1074,6008} -MaxEvents 10 | Format-List TimeCreated, Id, Message

# Blue screens: bugchecks on record
Get-WinEvent -FilterHashtable @{LogName='System'; ProviderName='Microsoft-Windows-WER-SystemErrorReporting'; Id=1001} -MaxEvents 5 | Format-List TimeCreated, Message

# Critical and error events, last 24 hours
Get-WinEvent -FilterHashtable @{LogName='System','Application'; Level=1,2; StartTime=(Get-Date).AddDays(-1)} | Select-Object TimeCreated, ProviderName, Id, Message

# CPU, free memory and disk latency, live (English counter names)
Get-Counter '\Processor(_Total)\% Processor Time', '\Memory\Available MBytes', '\LogicalDisk(_Total)\Avg. Disk sec/Transfer' -SampleInterval 2 -MaxSamples 5

# Processes with the most CPU time
Get-Process | Sort-Object CPU -Descending | Select-Object -First 10 Name, Id, CPU

# Processes with the most memory
Get-Process | Sort-Object WorkingSet64 -Descending | Select-Object -First 10 Name, Id, @{n='MB'; e={[int]($_.WorkingSet64 / 1MB)}}

# Automatic services that are not running
Get-CimInstance Win32_Service -Filter "StartMode='Auto' AND State<>'Running'" | Select-Object Name, DisplayName, ExitCode

# Free space per volume
Get-Volume | Where-Object DriveLetter | Select-Object DriveLetter, FileSystemLabel, @{n='FreeGB'; e={[int]($_.SizeRemaining / 1GB)}}, @{n='SizeGB'; e={[int]($_.Size / 1GB)}}

# Reboot pending (servicing or Windows Update)
'Component Based Servicing\RebootPending', 'WindowsUpdate\Auto Update\RebootRequired' | ForEach-Object { Test-Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\$_" }
```

## System

### systemd

```sh
# Status and last log lines
systemctl status <unit>

# Start now and at boot
systemctl enable --now <unit>

# Reload after editing unit files
systemctl daemon-reload

# Override a unit without touching the vendor file
systemctl edit <unit>

# Unit file as systemd sees it, drop-ins included
systemctl cat <unit>

# Effective limits and restart policy
systemctl show <unit> -p Restart,MemoryMax,LimitNOFILE,TasksMax

# After "start request repeated too quickly"
systemctl reset-failed <unit> && systemctl start <unit>

# Keep a unit from starting, even as a dependency
systemctl mask <unit>

# Check a unit file before loading it
systemd-analyze verify <file>.service

# Run one command with a memory and CPU cap
systemd-run --scope -p MemoryMax=2G -p CPUQuota=50% <command>

# CPU, memory and I/O per cgroup, live
systemd-cgtop

# Timers with last and next run
systemctl list-timers --all

# Every user's crontab (jobs outside systemd timers)
for u in $(cut -d: -f1 /etc/passwd); do crontab -l -u "$u" 2>/dev/null | sed "s/^/$u: /"; done

# What slowed down boot
systemd-analyze blame
```

### Logs

```sh
# Follow one unit
journalctl -u <unit> -f

# Last hour, all units
journalctl --since "1 hour ago"

# Exact time window
journalctl --since "2026-01-01 10:00" --until "2026-01-01 10:30"

# Previous boot (what happened before the crash)
journalctl -b -1 -e

# Boots on record, to pick one for -b
journalctl --list-boots

# Kernel messages only
journalctl -k

# Everything sudo ran
journalctl _COMM=sudo

# Keep logs across reboots
mkdir -p /var/log/journal && systemctl restart systemd-journald

# Follow a plain log file across rotation
tail -F <file>
```

### Processes

```sh
# Find a process by name
pgrep -a <name>

# Process tree
ps -ef --forest

# Files a process has open
lsof -p <pid>

# Who is using a port
ss -tlnp 'sport = :<port>'

# Environment of a running process
tr '\0' '\n' < /proc/<pid>/environ

# Binary and working directory of a process
ls -l /proc/<pid>/exe /proc/<pid>/cwd

# Kernel stack of a hung process
cat /proc/<pid>/stack

# Trace system calls
strace -f -p <pid>

# Only file and network syscalls, timestamped
strace -f -tt -e trace=%file,%network -p <pid>

# Hottest functions, live
perf top -p <pid>

# De-prioritise CPU and disk for a noisy job
renice -n 19 -p <pid> && ionice -c3 -p <pid>

# Watch a command, highlighting what changed
watch -n1 -d '<command>'

# Session that survives logout
tmux new -s <name>

# Reattach to it
tmux attach -t <name>
```

### Files and text

```sh
# Files changed in the last day
find <dir> -type f -mtime -1

# Delete files older than 30 days (run with -print first)
find <dir> -type f -mtime +30 -delete

# Top talkers in a log: count and rank one column
awk '{print $1}' <log> | sort | uniq -c | sort -rn | head

# "Operation not permitted" as root: immutable bit
lsattr <file> && chattr -i <file>

# Compare one file across two hosts
diff <(ssh <host-a> cat <file>) <(ssh <host-b> cat <file>)

# Run a command on many hosts in parallel
xargs -P 10 -I{} ssh -o BatchMode=yes {} '<command>' < <hosts-file>

# Pack a directory
tar -czf <archive>.tar.gz <dir>

# Unpack into a directory
tar -xzf <archive>.tar.gz -C <dir>

# List an archive without unpacking
tar -tzf <archive>.tar.gz | head
```

### Users and security

```sh
# Change owner recursively
chown -R <user>:<group> <dir>

# Directories 755, files 644
find <dir> -type d -exec chmod 755 {} + && find <dir> -type f -exec chmod 644 {} +

# Add a user to a group (next login)
usermod -aG <group> <user>

# Show ACLs
getfacl <path>

# Grant one user access via ACL
setfacl -m u:<user>:rwx <path>

# Default ACL so new files inherit group access
setfacl -R -d -m g:<group>:rwX <dir>

# Password and account expiry (service account locked out)
chage -l <user>

# SELinux denials, recent
ausearch -m avc -ts recent

# Reset SELinux labels after moving files
restorecon -Rv <path>

# Allow a service on a non-standard port under SELinux
semanage port -a -t http_port_t -p tcp <port>

# Failed SSH logins today
journalctl -u ssh -u sshd -g 'Failed password|Invalid user' --since today
```

### Packages

```sh
# Debian: installed and candidate version, and which repo
apt-cache policy <package>

# Debian: which package owns a file
dpkg -S <path>

# Debian: finish an interrupted upgrade
dpkg --configure -a && apt -f install

# Debian: pin a package version
apt-mark hold <package>

# Debian: reboot required, and by what
cat /var/run/reboot-required.pkgs

# Debian: services still running old libraries
needrestart -r l

# RHEL: which package owns a file
rpm -qf <path>

# RHEL: which package provides a binary
dnf provides '*/<binary>'

# RHEL: undo a transaction
dnf history && dnf history undo <id>

# RHEL: reboot required
dnf needs-restarting -r
```

### Boot and rescue

```sh
# Root is read-only in emergency mode
mount -o remount,rw /

# Chroot into a broken install from a live ISO
mount /dev/<root> /mnt && for d in dev proc sys run; do mount --rbind /$d /mnt/$d; done && chroot /mnt

# Kernel command line the system booted with
cat /proc/cmdline

# UEFI boot entries and order
efibootmgr -v

# Debian: rebuild every initramfs
update-initramfs -u -k all

# RHEL: rebuild every initramfs
dracut -f --regenerate-all

# Debian: reinstall GRUB on a BIOS disk
grub-install /dev/<disk> && update-grub
```

### Hardware

```sh
# Hardware event log (PSU, DIMM, fan faults)
ipmitool sel elist | tail -20

# Sensors: temperatures, fans, voltages
ipmitool sdr elist

# BMC network settings from the host
ipmitool lan print 1

# BMC unresponsive: cold-reset it from the host
ipmitool mc reset cold

# Serial number / Dell service tag
dmidecode -s system-serial-number

# DIMMs per slot with size and speed
dmidecode -t memory | grep -E 'Locator|Size|Speed'

# CPU model, sockets, cores, flags
lscpu

# ECC errors per memory controller
grep . /sys/devices/system/edac/mc/mc*/[cu]e_count

# PCIe link trained below its capability
lspci -vv -s <bus:dev.fn> | grep -E 'LnkCap|LnkSta'

# NUMA layout (pin VMs and IRQs to one node)
numactl --hardware

# CPU temperatures and fans
sensors
```

### NixOS: rebuild

```sh
# Build and switch
sudo nixos-rebuild switch --flake .#<host>

# Activate without a boot entry
sudo nixos-rebuild test --flake .#<host>

# Apply on next boot only (kernel, init changes)
sudo nixos-rebuild boot --flake .#<host>

# Build only, change nothing
nixos-rebuild build --flake .#<host>

# Deploy to a remote machine
nixos-rebuild switch --flake .#<host> --target-host <user>@<host> --sudo

# Back to the previous generation
sudo nixos-rebuild switch --rollback

# What changed since the last boot
nix store diff-closures /run/booted-system /run/current-system
```

### NixOS: flakes and store

```sh
# Update all inputs
nix flake update

# Update one input
nix flake update <input>

# Evaluate every output before deploying
nix flake check

# Evaluate an option
nix eval .#nixosConfigurations.<host>.config.<option>

# Explore the config in a REPL
nix repl --expr 'builtins.getFlake (toString ./.)'

# System generations
nix-env --list-generations --profile /nix/var/nix/profiles/system

# Closure size of the running system
nix path-info -Sh /run/current-system

# GC frees nothing: find what still roots it
nix-store --gc --print-roots | grep -v '^/proc'

# Delete old generations and garbage
sudo nix-collect-garbage --delete-older-than 14d

# Deduplicate the store
nix store optimise

# Store corruption: verify and repair
sudo nix-store --verify --check-contents --repair

# Try a package without installing
nix shell nixpkgs#<package>
```

### Windows: repair and updates

```powershell
# Repair the component store first (needs Windows Update or a source)
DISM /Online /Cleanup-Image /RestoreHealth

# Then repair system files
sfc /scannow

# What sfc could not repair
Select-String -Path C:\Windows\Logs\CBS\CBS.log -Pattern '\[SR\] Cannot repair'

# Shrink WinSxS by removing superseded components
DISM /Online /Cleanup-Image /StartComponentCleanup

# Recently installed updates
Get-HotFix | Sort-Object InstalledOn -Descending | Select-Object -First 10 HotFixID, Description, InstalledOn

# Find an update's package name to remove it
DISM /Online /Get-Packages /Format:Table | findstr <kb-number>

# Remove it
DISM /Online /Remove-Package /PackageName:<package-name>

# Readable Windows Update log (written to the desktop)
Get-WindowsUpdateLog

# Updates fail behind a proxy: the WinHTTP proxy setting
netsh winhttp show proxy

# Online scan of a volume, no downtime
Repair-Volume -DriveLetter <letter> -Scan
```

### Windows: services and tasks

```powershell
# Service stuck in Stopping: its PID (sc.exe, since sc is an alias)
sc.exe queryex <service>

# Then kill it
taskkill /PID <pid> /F

# Why a service will not start: Service Control Manager events
Get-WinEvent -FilterHashtable @{LogName='System'; ProviderName='Service Control Manager'} -MaxEvents 20 | Select-Object TimeCreated, Id, Message

# Scheduled tasks whose last run failed (ignoring running and never-run)
Get-ScheduledTask | Where-Object State -ne 'Disabled' | Get-ScheduledTaskInfo | Where-Object LastTaskResult -notin 0, 267009, 267011 | Select-Object TaskName, LastRunTime, LastTaskResult

# Which process holds a file (Sysinternals Handle)
handle.exe -accepteula <name>
```

### Windows: Active Directory

```powershell
# Replication summary across all DCs
repadmin /replsummary

# Replication partners and last result for one DC
repadmin /showrepl <dc>

# DC health across the forest, errors only
dcdiag /e /q

# Which DC this machine authenticates against
nltest /dsgetdc:<domain>

# FSMO role holders
netdom query fsmo

# "Trust relationship failed": repair the secure channel (Windows PowerShell 5.1, local admin)
Test-ComputerSecureChannel -Repair -Credential <domain>\<admin>

# Locked-out accounts
Search-ADAccount -LockedOut | Select-Object Name, SamAccountName, LastLogonDate

# Which machine keeps locking an account (event 4740 on the PDC)
Get-WinEvent -ComputerName (Get-ADDomain).PDCEmulator -FilterHashtable @{LogName='Security'; Id=4740} -MaxEvents 20 | Select-Object TimeCreated, @{n='User'; e={$_.Properties[0].Value}}, @{n='Caller'; e={$_.Properties[1].Value}}

# Unlock the account
Unlock-ADAccount -Identity <user>

# Applied Group Policy, as a report
gpresult /h C:\Temp\gpresult.html /f

# Reapply Group Policy now
gpupdate /force

# Kerberos tickets: list, then purge after a group change
klist; klist purge
```

## Storage

### Disks

```sh
# Block devices with filesystems and UUIDs
lsblk -f

# Model, serial, size, rotational, transport
lsblk -o NAME,SIZE,MODEL,SERIAL,ROTA,TRAN

# Logical and physical sector size (pick ashift)
lsblk -o NAME,LOG-SEC,PHY-SEC

# Stable device names
ls -l /dev/disk/by-id/

# Partition table
parted /dev/<disk> print

# Find a newly hot-added disk (SCSI host rescan)
echo "- - -" | tee /sys/class/scsi_host/host*/scan

# See a disk's new size after growing it (device rescan)
echo 1 > /sys/class/block/<disk>/device/rescan

# Same for NVMe namespaces
nvme ns-rescan /dev/<nvme-controller>

# Detach a disk cleanly before pulling it
echo 1 > /sys/block/<disk>/device/delete

# Re-read the partition table without rebooting
partprobe /dev/<disk>

# Grow a partition into the new space
growpart /dev/<disk> <partition-number>

# Find a physical drive: hold its activity LED on
dd if=/dev/<disk> of=/dev/null bs=1M status=progress

# Wipe filesystem, RAID and LVM signatures (destroys data)
wipefs -a /dev/<disk>

# Also clear GPT, including the backup at the end
sgdisk --zap-all /dev/<disk>

# "Device busy" on a wipe: leftover device-mapper entries
dmsetup ls && dmsetup remove <name>
```

### Filesystems and mounts

```sh
# Grow ext4 online
resize2fs /dev/<device>

# Grow XFS online (takes the mountpoint, not the device)
xfs_growfs <mountpoint>

# Force a full ext4 check (unmounted)
e2fsck -f /dev/<device>

# Repair XFS (fsck.xfs does nothing)
xfs_repair /dev/<device>

# Mount tree
findmnt
```

fstab entry that will not hang boot if the disk is missing:

```text
UUID=<uuid>  /mnt/<name>  ext4  defaults,noatime,nofail,x-systemd.device-timeout=10s  0 2
```

```sh
# Test fstab without rebooting
systemctl daemon-reload && findmnt --verify && mount -a

# Who keeps a mount busy
fuser -vm <mountpoint>

# Detach a busy mount
umount -l <mountpoint>

# Why a filesystem went read-only
dmesg -T | grep -iE 'remount|i/o error|ext4-fs error|xfs'

# Trim all mounted SSDs
fstrim -av
```

### Disk health and performance

```sh
# SMART verdict and attributes
smartctl -H -A /dev/<disk>

# SMART for a disk behind a PERC / MegaRAID
smartctl -a -d megaraid,<n> /dev/sda

# Start a long self-test
smartctl -t long /dev/<disk>

# Self-test results
smartctl -l selftest /dev/<disk>

# NVMe health and wear
nvme smart-log /dev/<nvme>

# Spin down a SATA disk now
hdparm -y /dev/<disk>

# Spin down a SAS disk now
sdparm --command=stop /dev/<disk>

# Live I/O per process
iotop -o

# I/O latency of a directory, like ping
ioping -c 10 <dir>

# Random 4k read/write benchmark
fio --name=rand --filename=<file> --size=1G --rw=randrw --bs=4k --direct=1 --runtime=30 --time_based --group_reporting
```

### NFS and SMB

```sh
# Server: apply /etc/exports changes
exportfs -ra

# Exports a server offers
showmount -e <server>

# Mount NFS
mount -t nfs4 <server>:/<export> <mountpoint>

# Mount options actually negotiated
nfsstat -m

# Stale NFS handle: drop the mount
umount -f -l <mountpoint>

# List SMB shares
smbclient -L //<server> -U <user>

# Mount SMB with a credentials file
mount -t cifs //<server>/<share> <mountpoint> -o credentials=/root/.smbcred,uid=<uid>,gid=<gid>,vers=3.0
```

### LVM

```sh
# Physical volumes, groups, logical volumes
pvs && vgs && lvs

# Which disks back each LV
lvs -a -o +devices

# Thin pool data and metadata fill
lvs -o lv_name,data_percent,metadata_percent <vg>

# Activate a VG after moving disks
vgchange -ay <vg>

# Metadata backups (undo a bad change)
vgcfgrestore --list <vg>

# Initialise a physical volume
pvcreate /dev/<partition>

# Create a volume group
vgcreate <vg> /dev/<partition>

# Fixed-size logical volume
lvcreate -n <lv> -L 50G <vg>

# Logical volume from all free space
lvcreate -n <lv> -l 100%FREE <vg>

# Thin pool
lvcreate --type thin-pool -n <pool> -L 100G <vg>
```

### LVM: grow and shrink

```sh
# Add a disk to a VG
vgextend <vg> /dev/<partition>

# PV after its disk was enlarged (rescan the disk first)
pvresize /dev/<partition>

# Grow LV and filesystem together
lvextend -r -L +10G /dev/<vg>/<lv>

# Grow into all free space
lvextend -r -l +100%FREE /dev/<vg>/<lv>

# Shrink LV and ext4 (unmounted; XFS cannot shrink)
lvreduce -r -L 20G /dev/<vg>/<lv>

# Grow a thin pool
lvextend -L +50G <vg>/<pool>

# Thin pool metadata filling up (full means corruption)
lvextend --poolmetadatasize +1G <vg>/<pool>

# Evacuate a disk
pvmove /dev/<partition>

# Then drop it from the VG
vgreduce <vg> /dev/<partition>
```

### LVM: snapshots and rescue

```sh
# Snapshot an LV
lvcreate -s -n <snap> -L 5G /dev/<vg>/<lv>

# Roll back to the snapshot
lvconvert --merge /dev/<vg>/<snap>

# Drop the snapshot
lvremove /dev/<vg>/<snap>

# Two VGs with the same name: rename one by UUID
vgs -o vg_name,vg_uuid && vgrename <vg-uuid> <new-name>

# Map partitions inside an LV (VM disk) to mount them
kpartx -av /dev/<vg>/<lv>
```

### ZFS: pools

```sh
# Only pools with problems
zpool status -x

# Files hit by permanent errors
zpool status -v <pool>

# Capacity per vdev
zpool list -v

# Mirror by stable disk IDs
zpool create -o ashift=12 <pool> mirror /dev/disk/by-id/<disk-a> /dev/disk/by-id/<disk-b>

# ashift of an existing pool
zdb -C <pool> | grep ashift

# Replace a failed disk
zpool replace <pool> <old-disk> /dev/disk/by-id/<new-disk>

# Use the space after replacing with bigger disks
zpool set autoexpand=on <pool> && zpool online -e <pool> <disk>

# Start a scrub
zpool scrub <pool>

# Reset error counters after fixing the cause
zpool clear <pool>

# Latency per vdev, live
zpool iostat -vl 2

# Who ran what against a pool
zpool history <pool> | tail -20

# Old ZFS label blocking reuse of a disk
zpool labelclear -f /dev/<disk>

# Find pools to import
zpool import

# Import by stable IDs
zpool import -d /dev/disk/by-id <pool>

# Import read-only for rescue
zpool import -o readonly=on -R /mnt <pool>
```

### ZFS: datasets and snapshots

```sh
# Usage per dataset
zfs list -o name,used,avail,refer,mountpoint

# Where the space went: data, snapshots, children
zfs list -o space -r <pool>

# Dataset with zstd compression
zfs create -o compression=zstd <pool>/<dataset>

# Compression ratio achieved
zfs get compressratio <pool>/<dataset>

# Only properties set by hand
zfs get -s local all <pool>/<dataset>

# Cap a dataset
zfs set quota=100G <pool>/<dataset>

# Snapshot recursively
zfs snapshot -r <pool>/<dataset>@<name>

# Snapshots, oldest first
zfs list -t snapshot -o name,used,creation -s creation

# Preview destroying a range of snapshots
zfs destroy -nv <pool>/<dataset>@<first>%<last>

# Roll back (destroys later snapshots)
zfs rollback -r <pool>/<dataset>@<name>

# Full send to another host
zfs send <pool>/<dataset>@<name> | ssh <host> zfs receive <pool>/<dataset>

# Incremental send, one step
zfs send -i @<old> <pool>/<dataset>@<new> | ssh <host> zfs receive <pool>/<dataset>

# Incremental send with every snapshot in between
zfs send -I @<old> <pool>/<dataset>@<new> | ssh <host> zfs receive <pool>/<dataset>
```

### ZFS: ARC

```sh
# ARC size and hit rate
arc_summary | head -40

# Live ARC stats
arcstat 1

# Cap ARC at 8 GB now
echo $((8 * 1024**3)) > /sys/module/zfs/parameters/zfs_arc_max

# Cap ARC at 8 GB across reboots
echo "options zfs zfs_arc_max=8589934592" > /etc/modprobe.d/zfs.conf && update-initramfs -u -k all
```

### Windows: files and storage

```powershell
# Mirror a folder with permissions, logged (/MIR deletes extras; add /L to preview)
robocopy <src> <dst> /MIR /COPY:DATS /DCOPY:DAT /MT:16 /R:1 /W:1 /LOG:C:\Temp\robocopy.log

# Largest files on a drive
Get-ChildItem <drive>:\ -Recurse -File -ErrorAction SilentlyContinue | Sort-Object Length -Descending | Select-Object -First 20 FullName, @{n='MB'; e={[int]($_.Length / 1MB)}}

# NTFS permissions
icacls <path>

# Access denied even as admin: take ownership, reset ACLs to inherited
takeown /F <path> /R /D Y; icacls <path> /reset /T /C

# New disk shows Offline (SAN policy): bring it online and writable
Set-Disk -Number <n> -IsOffline $false; Set-Disk -Number <n> -IsReadOnly $false

# Grew a VM disk: rescan, then extend the partition to the end
Update-HostStorageCache; Resize-Partition -DriveLetter <letter> -Size (Get-PartitionSupportedSize -DriveLetter <letter>).SizeMax

# Backup fails: VSS writers in a failed state
vssadmin list writers

# Shadow copy storage per volume
vssadmin list shadowstorage

# File server: who has a file open
Get-SmbOpenFile | Where-Object Path -like '*<name>*' | Select-Object ClientComputerName, ClientUserName, Path, FileId

# Close that handle
Close-SmbOpenFile -FileId <file-id> -Force

# Share permissions (NTFS permissions apply on top)
Get-SmbShareAccess -Name <share>
```

## Network

### Interfaces and routes

```sh
# Addresses, one line per interface
ip -br a

# Which route and source IP a destination takes
ip route get <ip>

# Policy routing (Tailscale, VPNs, multi-WAN add rules)
ip rule && ip route show table all

# ARP / neighbour table
ip neigh

# Add an address until reboot
ip addr add <ip>/<prefix> dev <iface>

# RX/TX errors and drops on an interface
ip -s link show <iface>

# Link speed and driver
ethtool <iface> && ethtool -i <iface>

# NIC counters: CRC errors point at cable or optic
ethtool -S <iface> | grep -iE 'err|drop|crc'

# Bond state and active slave
cat /proc/net/bonding/<bond>

# Bridge members
bridge link

# VLANs on a VLAN-aware bridge
bridge vlan show

# Someone else has this IP
arping -D -I <iface> -c 3 <ip>

# Reverse-path filter dropping asymmetric traffic
sysctl net.ipv4.conf.all.rp_filter net.ipv4.conf.<iface>.rp_filter

# Apply /etc/network/interfaces (Proxmox, ifupdown2)
ifreload -a

# NetworkManager: static IPv4 on a connection
nmcli con mod <connection> ipv4.method manual ipv4.addresses <ip>/<prefix> ipv4.gateway <gateway> && nmcli con up <connection>
```

### Ports and connections

```sh
# Listening sockets with processes
ss -tulpn

# Socket totals
ss -s

# Established connections on a port
ss -tnp state established '( sport = :<port> or dport = :<port> )'

# RTT, retransmits and window per connection
ss -ti dst <ip>

# Is a remote port open
nc -zv <host> <port>

# Talk to one backend, bypassing DNS and load balancer
curl -v --resolve <host>:443:<ip> https://<host>/

# HTTP status and timings
curl -o /dev/null -s -w '%{http_code} dns=%{time_namelookup}s connect=%{time_connect}s tls=%{time_appconnect}s total=%{time_total}s\n' <url>

# Certificate a server presents: subject, SANs, dates
openssl s_client -connect <host>:443 -servername <host> </dev/null 2>/dev/null | openssl x509 -noout -subject -ext subjectAltName -dates

# Expiry of a certificate file
openssl x509 -in <cert>.pem -noout -enddate
```

### DNS

```sh
# Just the answer
dig +short <name>

# Ask a specific resolver
dig @<server> <name> A

# What the system resolver returns (/etc/hosts, nsswitch)
getent hosts <name>

# Follow delegation from the root
dig +trace <name>

# Reverse lookup
dig -x <ip>

# systemd-resolved state per link
resolvectl status

# Flush the local cache
resolvectl flush-caches
```

### Path and packets

```sh
# Loss and latency per hop
mtr -rwc 50 <host>

# Largest packet that fits: 1472 for 1500 MTU, 1252 for 1280
ping -M do -s 1472 <host>

# Watch traffic on a port
tcpdump -ni <iface> port <port>

# Capture on every interface, bridges included
tcpdump -ni any host <ip>

# Capture a host to a file for Wireshark
tcpdump -ni <iface> -w <file>.pcap host <ip>

# Throughput test: server
iperf3 -s

# Throughput test: client, 4 streams
iperf3 -c <server> -P 4

# TCP retransmits and drops, system-wide
nstat -az | grep -iE 'retrans|drop'

# Bandwidth per connection
iftop -i <iface>
```

### Firewall and NAT

```sh
# nftables ruleset
nft list ruleset

# iptables with counters
iptables -L -n -v --line-numbers

# NAT rules (Docker and Kubernetes write here)
iptables -t nat -L -n -v

# Tracked connections to a host
conntrack -L -d <ip>

# Drop tracked state after changing rules
conntrack -D -d <ip>

# ufw status
ufw status verbose

# ufw: SSH from one subnet
ufw allow from <cidr> to any port 22 proto tcp

# firewalld zone rules
firewall-cmd --list-all

# firewalld: open a port
firewall-cmd --permanent --add-port=<port>/tcp && firewall-cmd --reload

# Enable IPv4 forwarding now
sysctl -w net.ipv4.ip_forward=1

# IPv4 forwarding across reboots
echo 'net.ipv4.ip_forward=1' > /etc/sysctl.d/99-forward.conf && sysctl --system
```

### Tailscale

```sh
# Peers and connection type
tailscale status

# Direct or relayed through DERP
tailscale ping <node>

# NAT type and DERP latency
tailscale netcheck

# Advertise a subnet
tailscale set --advertise-routes=<cidr>

# Accept subnets from other routers
tailscale set --accept-routes

# Route through an exit node
tailscale set --exit-node=<node>
```

### SSH

```sh
# New key pair
ssh-keygen -t ed25519 -C "<comment>"

# Install a public key on a host
ssh-copy-id -i ~/.ssh/id_ed25519.pub <user>@<host>

# Key auth refused: fix permissions
chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys
```

Kill a frozen session (press Enter first):

```text
~.
```

```sh
# Verbose handshake
ssh -vvv <user>@<host>

# Old switch, iDRAC or iLO rejects the key algorithm
ssh -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa <host>

# Forget a changed host key
ssh-keygen -R <host>

# Trust a host key non-interactively
ssh-keyscan -H <host> >> ~/.ssh/known_hosts

# Fingerprint of a server key
ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub

# Validate sshd_config before restarting
sshd -t

# Effective sshd setting for a given user
sshd -T -C user=<user> | grep -i <option>
```

### SSH tunnels

```sh
# Reach a remote-only port locally
ssh -N -L 8080:localhost:<remote-port> <host>

# Reach a web UI behind a bastion
ssh -N -L 8443:<target>:443 <bastion>

# Expose a local port on the remote
ssh -N -R 9000:localhost:<local-port> <host>

# SOCKS proxy through a host
ssh -N -D 1080 <host>

# Hop through a bastion
ssh -J <bastion> <host>
```

### Copy and transfer

```sh
# Mirror a directory, preserving everything
rsync -aHAX --info=progress2 --delete <src>/ <dst>/

# Preview what rsync would change
rsync -aHAXn --delete --itemize-changes <src>/ <dst>/

# Verify a copy by checksum: list files that differ
rsync -rcn --out-format='%n' <src>/ <dst>/

# Push over SSH, compressed
rsync -az --info=progress2 <src>/ <user>@<host>:<dst>/

# Resumable, throttled copy over SSH
rsync -aHAX --partial --bwlimit=50M --info=progress2 <src>/ <user>@<host>:<dst>/

# Stream a directory over SSH
tar -cf - <dir> | ssh <host> 'tar -xf - -C <dst>'

# Serve the current directory over HTTP
python3 -m http.server 8000
```

### Windows: network

```powershell
# Addresses, gateway and DNS per interface
Get-NetIPConfiguration

# Is a port reachable
Test-NetConnection <host> -Port <port>

# Trace the route
Test-NetConnection <host> -TraceRoute

# Resolve against a specific DNS server
Resolve-DnsName <name> -Server <server>

# Flush the DNS client cache
Clear-DnsClientCache

# Listening ports with their process
Get-NetTCPConnection -State Listen | Select-Object LocalAddress, LocalPort, @{n='Process'; e={(Get-Process -Id $_.OwningProcess).ProcessName}} | Sort-Object LocalPort

# Network marked Public blocks sharing and remoting: make it Private
Set-NetConnectionProfile -InterfaceAlias <interface> -NetworkCategory Private

# Firewall state per profile
Get-NetFirewallProfile | Select-Object Name, Enabled, DefaultInboundAction

# Allow an inbound port
New-NetFirewallRule -DisplayName "<name>" -Direction Inbound -Protocol TCP -LocalPort <port> -Action Allow

# Capture full packets with the built-in tool
pktmon start --capture --pkt-size 0 --file-name <file>.etl

# Stop, and convert the capture for Wireshark
pktmon stop; pktmon etl2pcap <file>.etl --out <file>.pcapng

# Time source and last sync
w32tm /query /status

# Clock offset against a domain controller
w32tm /stripchart /computer:<dc> /samples:5 /dataonly

# Force a time resync
w32tm /resync /force
```

### Windows: remote management

```powershell
# Is WinRM answering
Test-WSMan <host>

# Run a command on several servers
Invoke-Command -ComputerName <host-a>, <host-b> -ScriptBlock { <command> }

# Interactive remote session
Enter-PSSession -ComputerName <host>

# Who is logged on (RDP sessions)
quser /server:<host>

# Log off a stuck session
logoff <session-id> /server:<host>

# Enable RDP and its firewall rule
Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\Terminal Server' -Name fDenyTSConnections -Value 0; Enable-NetFirewallRule -DisplayGroup 'Remote Desktop'

# Installed software (not Win32_Product, which triggers MSI repairs)
Get-ItemProperty HKLM:\Software\Microsoft\Windows\CurrentVersion\Uninstall\*, HKLM:\Software\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\* | Where-Object DisplayName | Select-Object DisplayName, DisplayVersion | Sort-Object DisplayName

# Installed roles and features
Get-WindowsFeature | Where-Object Installed
```

## Virtualization

### Proxmox: VMs

```sh
# List VMs
qm list

# Clean shutdown
qm shutdown <vmid>

# Hard stop
qm stop <vmid>

# Stop a VM that holds a lock
qm stop <vmid> --skiplock

# QEMU process ignoring stop: kill it
kill -9 "$(cat /var/run/qemu-server/<vmid>.pid)"

# Clear a stale lock
qm unlock <vmid>

# Show config
qm config <vmid>

# Full KVM command line a VM starts with
qm showcmd <vmid> --pretty

# Serial console
qm terminal <vmid>

# Grow a disk (then rescan it inside the guest)
qm disk resize <vmid> scsi0 +10G

# Move a disk to another storage, live
qm disk move <vmid> scsi0 <storage> --delete 1

# Import a cloud image as a disk
qm disk import <vmid> <image>.qcow2 <storage>

# Import an OVF export
qm importovf <vmid> <file>.ovf <storage>

# Pick up orphaned disk images on storages
qm disk rescan --vmid <vmid>

# Snapshot
qm snapshot <vmid> <name>

# Roll back to a snapshot
qm rollback <vmid> <name>

# Full clone of a template
qm clone <template-id> <new-id> --name <name> --full

# Live migrate
qm migrate <vmid> <node> --online

# IPs via guest agent
qm guest cmd <vmid> network-get-interfaces

# Mount a stopped VM's partition from a ZFS zvol
mount /dev/zvol/<pool>/vm-<vmid>-disk-<n>-part<p> /mnt
```

### Proxmox: containers

```sh
# List containers
pct list

# Shell inside
pct enter <ctid>

# Run one command
pct exec <ctid> -- <command>

# Grow the root disk
pct resize <ctid> rootfs +5G

# Bind-mount a host directory
pct set <ctid> -mp0 <host-path>,mp=<container-path>

# Unprivileged bind mount: host UID = 100000 + container UID
chown -R 100000:100000 <host-path>

# Copy a file in
pct push <ctid> <file> <container-path>

# Mount a container's rootfs on the host (rescue)
pct mount <ctid>

# Check a container's filesystem
pct fsck <ctid>

# Clear a stale lock
pct unlock <ctid>
```

### Proxmox: host, storage and backup

```sh
# Versions of every PVE package
pveversion -v

# Core services
systemctl status pveproxy pvedaemon pve-cluster

# Web UI unreachable: restart its services
systemctl restart pveproxy pvedaemon

# Cluster quorum
pvecm status

# Lost quorum, /etc/pve read-only (last resort)
pvecm expected 1

# Regenerate node certificates
pvecm updatecerts --force

# ESPs in sync on a ZFS or UEFI root
proxmox-boot-tool status

# Upgrade (dist-upgrade; plain apt upgrade can break PVE)
apt update && apt dist-upgrade

# Storage pools and usage
pvesm status

# What is on a storage
pvesm list <storage>

# Back up now
vzdump <vmid> --storage <storage> --mode snapshot --compress zstd

# Restore a VM
qmrestore <backup>.vma.zst <new-id> --storage <storage>

# Restore a container
pct restore <new-id> <backup>.tar.zst --storage <storage>
```

### KVM: passthrough and libvirt

```sh
# CPU virtualization flags present
grep -Ec '(vmx|svm)' /proc/cpuinfo

# IOMMU enabled
dmesg | grep -e DMAR -e IOMMU

# Devices per IOMMU group
for d in /sys/kernel/iommu_groups/*/devices/*; do g=${d#*/iommu_groups/}; printf 'group %s\t' "${g%%/*}"; lspci -nns "${d##*/}"; done

# Driver bound to a device (vfio-pci or not)
lspci -nnk -s <bus:dev.fn>

# Nested virtualization on (Intel)
cat /sys/module/kvm_intel/parameters/nested

# libvirt domains
virsh list --all

# libvirt: guest IPs
virsh domifaddr <domain>

# Disks attached to a domain
virsh domblklist <domain>

# Image format and size
qemu-img info <disk>

# Convert VMDK to qcow2
qemu-img convert -p -f vmdk -O qcow2 <in>.vmdk <out>.qcow2
```

### Hyper-V

```powershell
# VMs with state, uptime and memory
Get-VM | Select-Object Name, State, Uptime, CPUUsage, MemoryAssigned

# Force a VM off
Stop-VM -Name <vm> -TurnOff

# VM stuck Stopping: kill its worker process
Get-CimInstance Win32_Process -Filter "Name='vmwp.exe'" | Where-Object CommandLine -match (Get-VM <vm>).VMId | ForEach-Object { Stop-Process -Id $_.ProcessId -Force }

# Checkpoints of a VM
Get-VMCheckpoint -VMName <vm>

# Delete all checkpoints, merging them into the disk
Get-VMCheckpoint -VMName <vm> | Remove-VMCheckpoint

# Grow a VHDX (then extend the partition in the guest)
Resize-VHD -Path <path>.vhdx -SizeBytes 200GB

# Virtual switches and the adapters behind them
Get-VMSwitch | Select-Object Name, SwitchType, NetAdapterInterfaceDescription
```

### ESXi: host and logs

```sh
# Version and build
vmware -vl

# Enable and start SSH from the ESXi shell
vim-cmd hostsvc/enable_ssh && vim-cmd hostsvc/start_ssh

# Enter maintenance mode
esxcli system maintenanceMode set --enable true

# Leave maintenance mode
esxcli system maintenanceMode set --enable false

# Reboot (host must be in maintenance mode)
esxcli system shutdown reboot --reason "<reason>"

# Host client or vCenter lost the host: restart management agents
/etc/init.d/hostd restart && /etc/init.d/vpxa restart

# Restart every management service (heavier)
services.sh restart

# Vendor, model and serial
esxcli hardware platform get

# NTP servers and state
esxcli system ntp get

# Back up the host configuration (prints a download URL)
vim-cmd hostsvc/firmware/sync_config && vim-cmd hostsvc/firmware/backup_config

# Support bundle for VMware / Broadcom
vm-support

# Storage, network and driver events
tail -f /var/log/vmkernel.log

# Host agent: VM operations, API calls
tail -f /var/log/hostd.log

# Hardware and connectivity observations
tail -f /var/log/vobd.log

# Why a VM failed to power on
tail -100 /vmfs/volumes/<datastore>/<vm>/vmware.log

# Send logs to a remote syslog
esxcli system syslog config set --loghost='udp://<ip>:514' && esxcli system syslog reload && esxcli network firewall ruleset set -r syslog -e true
```

### ESXi: VMs and snapshots

```sh
# Registered VMs with their IDs
vim-cmd vmsvc/getallvms

# Power state
vim-cmd vmsvc/power.getstate <vmid>

# Power on
vim-cmd vmsvc/power.on <vmid>

# Guest shutdown (needs VMware Tools)
vim-cmd vmsvc/power.shutdown <vmid>

# Hard power off
vim-cmd vmsvc/power.off <vmid>

# VM will not power off: find its world ID
esxcli vm process list

# Then kill it: soft, then hard, then force
esxcli vm process kill --type=soft --world-id=<world-id>

# Guest IP from VMware Tools
vim-cmd vmsvc/get.guest <vmid> | grep -m1 ipAddress

# Register a VM from its .vmx
vim-cmd solo/registervm /vmfs/volumes/<datastore>/<vm>/<vm>.vmx

# Unregister (files stay on the datastore)
vim-cmd vmsvc/unregister <vmid>

# Reload a .vmx after editing it by hand
vim-cmd vmsvc/reload <vmid>

# Stuck on a question ("moved or copied?"): list it
vim-cmd vmsvc/message <vmid>

# Answer it
vim-cmd vmsvc/message <vmid> <message-id> <choice>

# Snapshot tree
vim-cmd vmsvc/get.snapshotinfo <vmid>

# Create a snapshot without memory
vim-cmd vmsvc/snapshot.create <vmid> <name> <description> 0 0

# Delete all snapshots (consolidates into the base disk)
vim-cmd vmsvc/snapshot.removeall <vmid>

# Delta disks left behind
find /vmfs/volumes/<datastore>/<vm>/ -name '*-0000*.vmdk'
```

### ESXi: storage and virtual disks

```sh
# Datastores with capacity and free space
esxcli storage filesystem list

# See a new or grown LUN: rescan every adapter
esxcli storage core adapter rescan --all

# Refresh VMFS volumes after the rescan
vmkfstools -V

# Devices with their naa IDs
esxcli storage core device list

# Paths per device and their state
esxcli storage core path list

# Multipathing policy per device
esxcli storage nmp device list

# SMART data for a local disk
esxcli storage core device smart get -d <naa-id>

# Mount an NFS v3 datastore
esxcli storage nfs add -H <server> -s /<export> -v <datastore-name>

# Copied or replicated LUN not mounting: VMFS snapshot volumes
esxcli storage vmfs snapshot list

# Mount it keeping its signature
esxcli storage vmfs snapshot mount -l <volume-label>

# Reclaim freed blocks on thin-provisioned storage
esxcli storage vmfs unmap -l <datastore>

# Grow a VMDK to a new total size (VM off)
vmkfstools -X 100G /vmfs/volumes/<datastore>/<vm>/<vm>.vmdk

# Clone a disk as thin
vmkfstools -i <source>.vmdk -d thin <dest>.vmdk

# Return zeroed blocks of a thin disk (VM off)
vmkfstools -K <disk>.vmdk

# Check a VMDK for consistency
vmkfstools -x check <disk>.vmdk

# "File is locked": which host holds the lock (MAC in the output)
vmkfstools -D /vmfs/volumes/<datastore>/<vm>/<file>
```

### ESXi: network

```sh
# VMkernel interfaces with IPs
esxcli network ip interface ipv4 get

# Physical NICs: link, speed, driver
esxcli network nic list

# Driver and firmware of one NIC
esxcli network nic get -n <vmnic>

# Standard vSwitches and uplinks
esxcli network vswitch standard list

# Port groups and VLAN IDs
esxcli network vswitch standard portgroup list

# Set the VLAN on a port group
esxcli network vswitch standard portgroup set -p "<portgroup>" -v <vlan-id>

# Ping out a specific vmk, jumbo frames, no fragmenting
vmkping -I <vmk> -d -s 8972 <ip>

# Routes and neighbour table
esxcli network ip route ipv4 list && esxcli network ip neighbor list

# Which uplink and port a VM uses: find its world ID
esxcli network vm list

# Then its ports, MAC and team uplink
esxcli network vm port list -w <world-id>

# Capture on a vmk to a file
pktcap-uw --vmk <vmk> -o /tmp/<file>.pcap

# Capture on a physical uplink
pktcap-uw --uplink <vmnic> -o /tmp/<file>.pcap

# Management IP lost: set vmk0 from the shell
esxcli network ip interface ipv4 set -i vmk0 -t static -I <ip> -N <netmask> -g <gateway>

# Firewall rulesets
esxcli network firewall ruleset list
```

### ESXi: performance and patching

```sh
# Live stats: c CPU, m memory, d/u/v disk, n network
esxtop

# CPU contention: watch %RDY (>5% per vCPU) and %CSTP
esxtop   # then press c

# Disk latency: DAVG is the array, KAVG the host
esxtop   # then press u

# Record 5 minutes for later analysis
esxtop -b -d 5 -n 60 > /tmp/esxtop.csv

# Installed VIBs
esxcli software vib list

# Image profiles inside an offline depot
esxcli software sources profile list -d /vmfs/volumes/<datastore>/<depot>.zip

# Preview an update
esxcli software profile update -d /vmfs/volumes/<datastore>/<depot>.zip -p <profile> --dry-run

# Apply it (maintenance mode, then reboot)
esxcli software profile update -d /vmfs/volumes/<datastore>/<depot>.zip -p <profile>

# Install a driver or component VIB
esxcli software vib install -d /vmfs/volumes/<datastore>/<bundle>.zip

# Allow community-supported drivers
esxcli software acceptance set --level=CommunitySupported
```

### ESXi: PowerCLI

```powershell
# Connect to vCenter or a host
Connect-VIServer -Server <server>

# VMs with power state and size
Get-VM | Select-Object Name, PowerState, NumCpu, MemoryGB | Sort-Object Name

# Snapshots older than a week
Get-VM | Get-Snapshot | Where-Object { $_.Created -lt (Get-Date).AddDays(-7) } | Select-Object VM, Name, Created, SizeGB

# VMs whose disks need consolidation
Get-VM | Where-Object { $_.ExtensionData.Runtime.ConsolidationNeeded } | Select-Object Name

# Start SSH on every host
Get-VMHost | Get-VMHostService | Where-Object Key -eq "TSM-SSH" | Start-VMHostService
```

### ESXi to Proxmox

```sh
# Add the ESXi host as an import source on Proxmox
pvesm add esxi <storage-id> --server <esxi-host> --username root --password '<password>' --skip-cert-verification 1

# Before exporting: remove every snapshot
vim-cmd vmsvc/snapshot.removeall <vmid>

# Import a copied VMDK into a Proxmox VM
qm disk import <vmid> <vm>.vmdk <storage> --format raw

# Guest NIC renamed after the move (ens192 becomes ens18)
ip -br link
```

## Containers

### Docker

```sh
# Running, with status and ports
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'

# Follow logs
docker logs -f --tail 100 <container>

# Throwaway container
docker run --rm -it <image> sh

# Why it exited
docker inspect -f '{{.State.ExitCode}} oom={{.State.OOMKilled}} {{.State.Error}}' <container>

# Restart loops and other events, last 10 minutes
docker events --since 10m

# Container IPs
docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' <container>

# CPU and memory snapshot
docker stats --no-stream

# Log file on disk (the one filling /var)
docker inspect -f '{{.LogPath}}' <container>

# Writable layer size per container
docker ps -s

# Files changed inside a container
docker diff <container>

# Copy a file out
docker cp <container>:<path> <dest>

# Change restart policy
docker update --restart unless-stopped <container>

# Host PID of every container
docker ps -q | xargs docker inspect -f '{{.State.Pid}} {{.Name}}'

# Run host tools inside a container's network namespace
nsenter -t "$(docker inspect -f '{{.State.Pid}}' <container>)" -n ss -tlnp
```

### Docker Compose

```sh
# Update images and recreate
docker compose pull && docker compose up -d

# Rebuild and recreate
docker compose up -d --build

# Recreate one service even if unchanged
docker compose up -d --force-recreate <service>

# Follow one service
docker compose logs -f <service>

# Final config after env files and merges
docker compose config

# Every compose project on the host
docker compose ls

# Tear down, volumes too (destroys data)
docker compose down -v
```

### Docker: images and cleanup

```sh
# Disk used by images, containers, volumes, build cache
docker system df -v

# Remove everything unused except volumes
docker system prune -a

# Build cache (often the biggest)
docker builder prune

# Unused images older than a week
docker image prune -a --filter "until=168h"

# Remove unused anonymous volumes
docker volume prune

# Multi-arch build and push
docker buildx build --platform linux/amd64,linux/arm64 -t <name>:<tag> --push .

# Architectures and digest of a remote image
docker buildx imagetools inspect <image>

# Layers and their sizes
docker history <image>

# Export an image for an offline host
docker save <image> | gzip > <image>.tar.gz

# Import it
docker load -i <image>.tar.gz
```

### Docker: daemon, networks, volumes

```sh
# Networks and their members
docker network inspect <network>

# Debug a container's network with a full toolbox
docker run --rm -it --network container:<container> nicolaka/netshoot

# Publish on localhost only (published ports bypass ufw)
docker run -d -p 127.0.0.1:<host-port>:<container-port> <image>

# Back up a named volume
docker run --rm -v <volume>:/data -v "$PWD":/backup alpine tar -czf /backup/<volume>.tar.gz -C /data .
```

Cap container logs (/etc/docker/daemon.json):

```json
{ "log-driver": "json-file", "log-opts": { "max-size": "10m", "max-file": "3" } }
```

Keep Docker networks off your LAN ranges (daemon.json):

```json
{ "default-address-pools": [{ "base": "10.200.0.0/16", "size": 24 }] }
```

### Kubernetes: inspect

```sh
# Switch cluster
kubectl config use-context <context>

# Default namespace for this context
kubectl config set-context --current --namespace=<ns>

# Field docs for any resource
kubectl explain <resource>.spec

# Pods not running, cluster-wide
kubectl get pods -A --field-selector=status.phase!=Running

# Why a pod is stuck
kubectl describe pod <pod>

# Events, newest last
kubectl get events -A --sort-by=.metadata.creationTimestamp

# Warnings only
kubectl get events -A --field-selector type=Warning --sort-by=.metadata.creationTimestamp

# Events for one object
kubectl get events -n <ns> --field-selector involvedObject.name=<name>

# Pods by restart count
kubectl get pods -A --sort-by='.status.containerStatuses[0].restartCount'

# Why the last container died (OOMKilled, exit code)
kubectl get pod <pod> -o jsonpath='{.status.containerStatuses[*].lastState}'

# Node usage
kubectl top nodes

# Heaviest pods
kubectl top pods -A --sort-by=memory

# Requests and limits against what a node has
kubectl describe node <node> | grep -A 10 'Allocated resources'

# Service with no endpoints: selector matches nothing
kubectl get endpointslices -l kubernetes.io/service-name=<service>

# Decode a secret key
kubectl get secret <name> -o jsonpath='{.data.<key>}' | base64 -d

# Decode every key of a secret
kubectl get secret <name> -o json | jq '.data | map_values(@base64d)'

# Every image running
kubectl get pods -A -o jsonpath='{range .items[*]}{.spec.containers[*].image}{"\n"}{end}' | sort -u

# API server health, check by check
kubectl get --raw='/readyz?verbose'
```

### Kubernetes: debug

```sh
# Follow a container's logs
kubectl logs -f <pod> -c <container>

# Logs from before the crash
kubectl logs --previous <pod>

# Logs across a label
kubectl logs -l app=<app> --all-containers --prefix --since=1h

# Shell inside
kubectl exec -it <pod> -- sh

# Debug container in a distroless pod
kubectl debug -it <pod> --image=busybox --target=<container>

# Shell on a node, host filesystem at /host
kubectl debug node/<node> -it --image=busybox

# Forward a service locally
kubectl port-forward svc/<service> 8080:<port>

# Cluster DNS works?
kubectl run dns-test --rm -it --image=busybox:1.36 --restart=Never -- nslookup kubernetes.default

# Copy a file out of a pod
kubectl cp <ns>/<pod>:<path> <dest>

# What a service account can do
kubectl auth can-i --list --as=system:serviceaccount:<ns>:<sa>

# Containers on a node without the API
crictl ps -a
```

### Kubernetes: workloads and nodes

```sh
# Restart pods of a deployment
kubectl rollout restart deploy/<name>

# Wait for a rollout
kubectl rollout status deploy/<name>

# Undo the last rollout
kubectl rollout undo deploy/<name>

# Scale to zero
kubectl scale deploy/<name> --replicas=0

# See what apply would change
kubectl diff -f <file>.yaml

# YAML skeleton without creating anything
kubectl create deploy <name> --image=<image> --dry-run=client -o yaml

# Drain for maintenance
kubectl drain <node> --ignore-daemonsets --delete-emptydir-data

# Back into service
kubectl uncordon <node>

# Pods on one node
kubectl get pods -A -o wide --field-selector spec.nodeName=<node>
```

### Kubernetes: stuck resources

```sh
# Namespace stuck Terminating: what is left in it
kubectl api-resources --verbs=list --namespaced -o name | xargs -n1 kubectl get -n <ns> --ignore-not-found --show-kind

# Resource held by finalizers
kubectl patch <kind> <name> -n <ns> --type=merge -p '{"metadata":{"finalizers":null}}'

# Pod stuck Terminating on a dead node
kubectl delete pod <pod> --grace-period=0 --force

# Volume will not attach: stale attachments
kubectl get volumeattachment

# Clear out evicted and failed pods
kubectl delete pods -A --field-selector=status.phase=Failed

# kubeadm: certificate expiry
kubeadm certs check-expiration
```

### Helm

```sh
# Add and refresh a repo
helm repo add <repo> <url> && helm repo update

# Chart's default values
helm show values <repo>/<chart>

# Install or upgrade
helm upgrade --install <release> <repo>/<chart> -n <ns> --create-namespace -f values.yaml

# Render manifests locally
helm template <release> <repo>/<chart> -f values.yaml

# Releases stuck pending or failed
helm list -A --pending --failed

# Values a release was deployed with
helm get values <release> -n <ns>

# Manifests a release actually applied
helm get manifest <release> -n <ns>

# Revisions of a release
helm history <release> -n <ns>

# Roll back (also unsticks pending-upgrade)
helm rollback <release> <revision> -n <ns>
```

### Flux

```sh
# Controllers healthy
flux check

# Everything Flux manages
flux get all -A

# Only what is failing
flux get all -A --status-selector ready=false

# Pull git and apply now
flux reconcile kustomization flux-system --with-source

# Reconcile one Helm release
flux reconcile helmrelease <name> -n <ns> --with-source

# Local changes against the cluster, before pushing
flux diff kustomization <name> --path <path>

# Pause GitOps for manual work
flux suspend kustomization <name>

# Resume it
flux resume kustomization <name>

# Controller errors
flux logs -A --level=error
```

### Talos

```sh
# Cluster health from one node
talosctl -n <ip> health

# Live node dashboard
talosctl -n <ip> dashboard

# Services and their state
talosctl -n <ip> services

# Kubelet logs
talosctl -n <ip> logs kubelet

# Kernel log
talosctl -n <ip> dmesg

# Pod containers on a node
talosctl -n <ip> containers -k

# Disks Talos sees
talosctl -n <ip> get disks

# etcd members
talosctl -n <ip> etcd members

# etcd health and leader
talosctl -n <ip> etcd status

# Snapshot etcd
talosctl -n <ip> etcd snapshot <file>.snapshot

# Apply a changed machine config
talosctl -n <ip> apply-config -f <node>.yaml

# Patch the machine config
talosctl -n <ip> patch mc --patch @<patch>.yaml

# Upgrade Talos on a node
talosctl -n <ip> upgrade --image <installer-image>:<version>

# Preview a Kubernetes upgrade
talosctl -n <ip> upgrade-k8s --to <version> --dry-run

# Upgrade Kubernetes
talosctl -n <ip> upgrade-k8s --to <version>

# Fetch kubeconfig
talosctl -n <ip> kubeconfig
```