From 0d4f700b89075957ec6b4a60512dda3fda42fe7b Mon Sep 17 00:00:00 2001 From: Shahab Dogar Date: Sat, 19 Oct 2024 13:15:36 +0500 Subject: [PATCH] feat: NixOS | use LUKS and proper hostname for k3s domain feat: NixOS | move cluster config to /master fix: update all stuff for office network feat: PiHole | set up DHCP server chore: Cloudflare | delete api token secret chore: remove external-dns annotations from ingressed services fix: PiHole | turn off liveness checks due to host ip fix: GiteaActions | use encrypted storage for runner fix: ElasticSearch | use encrypted volumes for storage fix: Pihole | static mac addresses all caps feat: NixOS | manual network configuration fix: NixOS | k3s cluster init point to static ip with tls-san chore: Postgres | move certificate resources into own file + reduce volume size fix: Pihole | add ingress class name sec: NixOS | remove token from git --- 1password/secrets.yaml | 20 +-- elasticsearch/cluster.yaml | 10 ++ gitea/actions/runner.yaml | 2 +- helm/helmfile.yaml | 2 + helm/values/gitea.values.yaml | 3 +- helm/values/longhorn.values.yaml | 13 ++ helm/values/nginx-internal.values.yaml | 3 + helm/values/pihole.values.yaml | 49 +++++++- helm/values/prometheus.values.yaml | 3 - kustomize/metallb/pool.yaml | 2 +- longhorn/encrypted-storage-class.yaml | 18 +++ nixos/disko-config.nix | 56 --------- nixos/{ => master}/configuration.nix | 21 +++- nixos/master/disko-config.nix | 40 ++++++ nixos/{ => master}/flake.lock | 0 nixos/{ => master}/flake.nix | 0 nixos/{ => master}/hardware-configuration.nix | 4 +- postgres/certificates.yaml | 110 +++++++++++++++++ postgres/cluster.yaml | 115 +----------------- 19 files changed, 280 insertions(+), 191 deletions(-) create mode 100644 helm/values/longhorn.values.yaml create mode 100644 longhorn/encrypted-storage-class.yaml delete mode 100644 nixos/disko-config.nix rename nixos/{ => master}/configuration.nix (88%) create mode 100644 nixos/master/disko-config.nix rename nixos/{ => master}/flake.lock (100%) rename nixos/{ => master}/flake.nix (100%) rename nixos/{ => master}/hardware-configuration.nix (91%) diff --git a/1password/secrets.yaml b/1password/secrets.yaml index 9ae579b..03f2e7f 100644 --- a/1password/secrets.yaml +++ b/1password/secrets.yaml @@ -61,16 +61,6 @@ spec: --- apiVersion: onepassword.com/v1 kind: OnePasswordItem -metadata: - name: cloudflare-token - namespace: cloudflare-system - annotations: - operator.1password.io/auto-restart: "true" -spec: - itemPath: "vaults/Lab/items/Cloudflare" ---- -apiVersion: onepassword.com/v1 -kind: OnePasswordItem metadata: name: authentik-postgres namespace: authentik-system @@ -138,3 +128,13 @@ metadata: operator.1password.io/auto-restart: "true" spec: itemPath: "vaults/Lab/items/smtp-token" +--- +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: longhorn-encryption + namespace: longhorn-system + annotations: + operator.1password.io/auto-restart: "true" +spec: + itemPath: "vaults/Lab/items/longhorn-encryption" diff --git a/elasticsearch/cluster.yaml b/elasticsearch/cluster.yaml index b66c7bb..24157cc 100644 --- a/elasticsearch/cluster.yaml +++ b/elasticsearch/cluster.yaml @@ -38,5 +38,15 @@ spec: nodeSets: - name: master count: 1 + volumeClaimTemplates: + - metadata: + name: elasticsearch-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn-crypto config: node.roles: ["master", "data"] diff --git a/gitea/actions/runner.yaml b/gitea/actions/runner.yaml index da0ca76..7aafdff 100644 --- a/gitea/actions/runner.yaml +++ b/gitea/actions/runner.yaml @@ -9,7 +9,7 @@ spec: resources: requests: storage: 10Gi - storageClassName: longhorn + storageClassName: longhorn-crypto --- apiVersion: apps/v1 kind: Deployment diff --git a/helm/helmfile.yaml b/helm/helmfile.yaml index 9158e03..668e445 100644 --- a/helm/helmfile.yaml +++ b/helm/helmfile.yaml @@ -26,6 +26,8 @@ releases: namespace: longhorn-system chart: longhorn/longhorn version: 1.7.0 + values: + - ./values/longhorn.values.yaml # Load Balancer - name: metallb diff --git a/helm/values/gitea.values.yaml b/helm/values/gitea.values.yaml index 36c5e61..c764985 100644 --- a/helm/values/gitea.values.yaml +++ b/helm/values/gitea.values.yaml @@ -1,11 +1,9 @@ service: http: annotations: - external-dns.alpha.kubernetes.io/hostname: git.dogar.dev metallb.universe.tf/allow-shared-ip: gitea ssh: annotations: - external-dns.alpha.kubernetes.io/hostname: git.dogar.dev metallb.universe.tf/allow-shared-ip: gitea ingress: enabled: true @@ -88,6 +86,7 @@ gitea: key: password persistence: enabled: true + storageClass: longhorn-crypto accessModes: - ReadWriteMany postgresql-ha: diff --git a/helm/values/longhorn.values.yaml b/helm/values/longhorn.values.yaml new file mode 100644 index 0000000..3d8904b --- /dev/null +++ b/helm/values/longhorn.values.yaml @@ -0,0 +1,13 @@ +metrics: + serviceMonitor: + enabled: true +ingress: + enabled: true + ingressClassName: nginx-internal + host: longhorn.dogar.dev + tls: true + tlsSecretName: longhorn-tls + annotations: + cert-manager.io/cluster-issuer: cloudflare-issuer + cert-manager.io/acme-challenge-type: dns01 + cert-manager.io/private-key-size: "4096" diff --git a/helm/values/nginx-internal.values.yaml b/helm/values/nginx-internal.values.yaml index 69232db..eaa332a 100644 --- a/helm/values/nginx-internal.values.yaml +++ b/helm/values/nginx-internal.values.yaml @@ -6,6 +6,9 @@ controller: controllerValue: "k8s.io/ingress-nginx" parameters: {} ingressClass: nginx-internal + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: "postgres.dogar.dev" tcp: 22: "gitea-system/gitea-ssh:22" 5432: "postgres-system/postgres-cluster-rw:5432" diff --git a/helm/values/pihole.values.yaml b/helm/values/pihole.values.yaml index 61b71a7..d53e9bb 100644 --- a/helm/values/pihole.values.yaml +++ b/helm/values/pihole.values.yaml @@ -2,17 +2,21 @@ DNS1: 1.1.1.1 DNS2: - 192.168.0.1 + 192.168.18.1 +nodeSelector: + pihole: "true" admin: enabled: true existingSecret: pihole-admin passwordKey: password persistentVolumeClaim: enabled: true + storageClass: longhorn-crypto accessModes: - ReadWriteOnce ingress: enabled: true + ingressClassName: nginx-internal annotations: cert-manager.io/cluster-issuer: cloudflare-issuer cert-manager.io/acme-challenge-type: dns01 @@ -24,15 +28,54 @@ ingress: hosts: - pihole.dogar.dev serviceWeb: - loadBalancerIP: 192.168.0.250 annotations: metallb.universe.tf/allow-shared-ip: pihole-svc type: LoadBalancer + loadBalancerIP: 192.168.18.250 serviceDns: - loadBalancerIP: 192.168.0.250 annotations: metallb.universe.tf/allow-shared-ip: pihole-svc type: LoadBalancer + loadBalancerIP: 192.168.18.250 +serviceDhcp: + annotations: + metallb.universe.tf/allow-shared-ip: pihole-svc + enabled: true + type: LoadBalancer + loadBalancerIP: 192.168.18.250 +probes: + liveness: + enabled: false + readiness: + enabled: false +dnsmasq: + additionalHostsEntries: + - 192.168.18.10 homelab-0 + - 192.168.18.11 homelab-1 + - 192.168.18.12 homelab-2 + - 192.168.18.10 lab.dogar.dev + staticDhcpEntries: + - dhcp-host=B0:41:6F:0F:A8:D3,192.168.18.10,homelab-0 + - dhcp-host=B0:41:6F:0F:AE:89,192.168.18.11,homelab-1 + - dhcp-host=B0:41:6F:0F:A0:CD,192.168.18.12,homelab-2 +hostNetwork: true +hostname: pihole +privileged: true +capabilities: + add: + - NET_ADMIN +extraEnvVars: + TZ: "Asia/Karachi" + DNSSEC: "true" + FTLCONF_LOCAL_IPV4: "192.168.18.250" + INTERFACE: "enp1s0" + DNSMASQ_LISTENING: "single" + DHCP_ACTIVE: "true" + DHCP_START: "192.168.18.2" + DHCP_END: "192.168.18.20" + DHCP_ROUTER: "192.168.18.1" + PIHOLE_DOMAIN: "pihole.dogar.dev" + VIRTUAL_HOST: "pihole.dogar.dev" podAnnotations: prometheus.io/scrape: "true" prometheus.io/port: "9617" diff --git a/helm/values/prometheus.values.yaml b/helm/values/prometheus.values.yaml index 0dea4b7..0495cb6 100644 --- a/helm/values/prometheus.values.yaml +++ b/helm/values/prometheus.values.yaml @@ -1,8 +1,5 @@ grafana: enabled: true - service: - annotations: - external-dns.alpha.kubernetes.io/hostname: grafana.dogar.dev ingress: enabled: true ingressClassName: nginx-internal diff --git a/kustomize/metallb/pool.yaml b/kustomize/metallb/pool.yaml index 6cd1080..1fb8151 100644 --- a/kustomize/metallb/pool.yaml +++ b/kustomize/metallb/pool.yaml @@ -6,7 +6,7 @@ metadata: namespace: metallb-system spec: addresses: - - 192.168.0.192/26 + - 192.168.18.192/26 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement diff --git a/longhorn/encrypted-storage-class.yaml b/longhorn/encrypted-storage-class.yaml new file mode 100644 index 0000000..08b612d --- /dev/null +++ b/longhorn/encrypted-storage-class.yaml @@ -0,0 +1,18 @@ +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: longhorn-crypto + namespace: longhorn-system +provisioner: driver.longhorn.io +allowVolumeExpansion: true +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "2880" # 48 hours in minutes + encrypted: "true" + csi.storage.k8s.io/provisioner-secret-name: "longhorn-encryption" + csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" + csi.storage.k8s.io/node-publish-secret-name: "longhorn-encryption" + csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" + csi.storage.k8s.io/node-stage-secret-name: "longhorn-encryption" + csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" diff --git a/nixos/disko-config.nix b/nixos/disko-config.nix deleted file mode 100644 index 36219d9..0000000 --- a/nixos/disko-config.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ - disko.devices = { - disk = { - vdb = { - type = "disk"; - device = "/dev/nvme0n1"; - content = { - type = "gpt"; - partitions = { - ESP = { - priority = 1; - name = "ESP"; - start = "1M"; - end = "128M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; - }; - }; - root = { - size = "100%"; - content = { - type = "btrfs"; - extraArgs = [ "-f" ]; # Override existing partition - # Subvolumes must set a mountpoint in order to be mounted, - # unless their parent is mounted - subvolumes = { - # Subvolume name is different from mountpoint - "/rootfs" = { - mountpoint = "/"; - }; - # Subvolume name is the same as the mountpoint - "/home" = { - mountOptions = [ "compress=zstd" ]; - mountpoint = "/home"; - }; - # Sub(sub)volume doesn't need a mountpoint as its parent is mounted - "/home/shahab" = { }; - # Parent is not mounted so the mountpoint must be set - "/nix" = { - mountOptions = [ "compress=zstd" "noatime" ]; - mountpoint = "/nix"; - }; - }; - - mountpoint = "/partition-root"; - }; - }; - }; - }; - }; - }; - }; -} diff --git a/nixos/configuration.nix b/nixos/master/configuration.nix similarity index 88% rename from nixos/configuration.nix rename to nixos/master/configuration.nix index 628f5c5..4e9c0ce 100644 --- a/nixos/configuration.nix +++ b/nixos/master/configuration.nix @@ -23,6 +23,22 @@ networking.hostName = meta.hostname; # Define your hostname. # Pick only one of the below networking options. networking.networkmanager.enable = true; + networking.interfaces.enp1s0.ipv4.addresses = [ + { + address = ( + if meta.hostname == "homelab-0" then "192.168.18.10" + else if meta.hostname == "homelab-1" then "192.168.18.11" + else if meta.hostname == "homelab-2" then "192.168.18.12" + else throw "Unknown hostname" + ); + prefixLength = 24; + } + ]; + networking.defaultGateway = "192.168.18.1"; + networking.nameservers = [ + "192.168.18.250" + "1.1.1.1" + ]; # Set your time zone. time.timeZone = "Asia/Karachi"; @@ -61,8 +77,9 @@ "--disable servicelb" "--disable traefik" "--disable local-storage" + "--tls-san homelab-0" ] ++ (if meta.hostname == "homelab-0" then [] else [ - "--server https://homelab-0:6443" + "--server https://192.168.18.10:6443" ])); clusterInit = (meta.hostname == "homelab-0"); }; @@ -147,6 +164,6 @@ # and migrated your data accordingly. # # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion . - system.stateVersion = "23.11"; # Did you read the comment? + system.stateVersion = "24.05"; # Did you read the comment? } diff --git a/nixos/master/disko-config.nix b/nixos/master/disko-config.nix new file mode 100644 index 0000000..d12e051 --- /dev/null +++ b/nixos/master/disko-config.nix @@ -0,0 +1,40 @@ +{ + disko.devices = { + disk = { + vdb = { + type = "disk"; + device = "/dev/nvme0n1"; + content = { + type = "gpt"; + partitions = { + ESP = { + priority = 1; + name = "ESP"; + start = "1M"; + end = "128M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + luks = { + size = "100%"; + content = { + name = "crypted"; + type = "luks"; + askPassword = true; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/nixos/flake.lock b/nixos/master/flake.lock similarity index 100% rename from nixos/flake.lock rename to nixos/master/flake.lock diff --git a/nixos/flake.nix b/nixos/master/flake.nix similarity index 100% rename from nixos/flake.nix rename to nixos/master/flake.nix diff --git a/nixos/hardware-configuration.nix b/nixos/master/hardware-configuration.nix similarity index 91% rename from nixos/hardware-configuration.nix rename to nixos/master/hardware-configuration.nix index 9512663..53c8c9d 100644 --- a/nixos/hardware-configuration.nix +++ b/nixos/master/hardware-configuration.nix @@ -15,8 +15,8 @@ # (the default) this is the recommended approach. When using systemd-networkd it's # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0.useDHCP = lib.mkDefault true; + networking.useDHCP = lib.mkDefault false; + networking.interfaces.enp1s0.useDHCP = lib.mkDefault false; # networking.interfaces.wlo1.useDHCP = lib.mkDefault true; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; diff --git a/postgres/certificates.yaml b/postgres/certificates.yaml index 56d3361..698356b 100644 --- a/postgres/certificates.yaml +++ b/postgres/certificates.yaml @@ -1,5 +1,115 @@ --- apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: postgres-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: server-ca + namespace: postgres-system +spec: + isCA: true + commonName: postgres-server-ca + secretName: postgres-server-ca + privateKey: + algorithm: ECDSA + size: 384 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: postgres-server-ca-issuer + namespace: postgres-system +spec: + ca: + secretName: postgres-server-ca +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-server-cert + namespace: postgres-system + labels: + cnpg.io/reload: "" +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: postgres-server-cert + namespace: postgres-system +spec: + secretName: postgres-server-cert + usages: + - server auth + dnsNames: + - postgres-cluster-rw.postgres-system.svc.cluster.local + - postgres-cluster-ro.postgres-system.svc.cluster.local + - postgres-cluster-r.postgres-system.svc.cluster.local + - postgres.dogar.dev + issuerRef: + name: postgres-server-ca-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: client-ca + namespace: postgres-system +spec: + isCA: true + commonName: postgres-client-ca + secretName: postgres-client-ca + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: postgres-client-ca-issuer + namespace: postgres-system +spec: + ca: + secretName: postgres-client-ca +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-client-cert + namespace: postgres-system + labels: + cnpg.io/reload: "" +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: postgres-client-cert + namespace: postgres-system +spec: + secretName: postgres-client-cert + usages: + - client auth + commonName: streaming_replica + issuerRef: + name: postgres-client-ca-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: shahab-client-cert diff --git a/postgres/cluster.yaml b/postgres/cluster.yaml index 08bd079..0bed2f2 100644 --- a/postgres/cluster.yaml +++ b/postgres/cluster.yaml @@ -1,114 +1,4 @@ --- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: postgres-system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: server-ca - namespace: postgres-system -spec: - isCA: true - commonName: postgres-server-ca - secretName: postgres-server-ca - privateKey: - algorithm: ECDSA - size: 384 - issuerRef: - name: selfsigned-issuer - kind: Issuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: postgres-server-ca-issuer - namespace: postgres-system -spec: - ca: - secretName: postgres-server-ca ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgres-server-cert - namespace: postgres-system - labels: - cnpg.io/reload: "" ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: postgres-server-cert - namespace: postgres-system -spec: - secretName: postgres-server-cert - usages: - - server auth - dnsNames: - - postgres-cluster-rw.postgres-system.svc.cluster.local - - postgres-cluster-ro.postgres-system.svc.cluster.local - - postgres-cluster-r.postgres-system.svc.cluster.local - - postgres.dogar.dev - issuerRef: - name: postgres-server-ca-issuer - kind: Issuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: client-ca - namespace: postgres-system -spec: - isCA: true - commonName: postgres-client-ca - secretName: postgres-client-ca - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: selfsigned-issuer - kind: Issuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: postgres-client-ca-issuer - namespace: postgres-system -spec: - ca: - secretName: postgres-client-ca ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgres-client-cert - namespace: postgres-system - labels: - cnpg.io/reload: "" ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: postgres-client-cert - namespace: postgres-system -spec: - secretName: postgres-client-cert - usages: - - client auth - commonName: streaming_replica - issuerRef: - name: postgres-client-ca-issuer - kind: Issuer - group: cert-manager.io ---- apiVersion: postgresql.cnpg.io/v1 kind: Cluster metadata: @@ -127,7 +17,6 @@ spec: pg_hba: - hostssl all shahab all cert - hostssl sameuser all all cert - - hostssl giteadb gitea 10.42.0.0/16 scram-sha-256 enableSuperuserAccess: false bootstrap: initdb: @@ -137,4 +26,8 @@ spec: postInitSQL: - 'CREATE USER shahab SUPERUSER;' storage: + size: 10Gi + storageClass: longhorn-crypto + walStorage: size: 1Gi + storageClass: longhorn-crypto