feat: NixCache | pull nix cache out into its own stack

cert-manager/certificate.ts

@@ -0,0 +1,85 @@
+import { Construct } from "constructs";
+import { Manifest } from "@cdktf/provider-kubernetes/lib/manifest";
+import { KubernetesProvider } from "@cdktf/provider-kubernetes/lib/provider";
+
+export interface CertificateOptions {
+  provider: KubernetesProvider;
+
+  /** Namespace to create the Certificate in */
+  namespace: string;
+
+  /** Required name of the certificate (and CRD name) */
+  name: string;
+
+  /** Secret name for storing the issued TLS cert */
+  secretName: string;
+
+  /** One or more DNS names the certificate should cover */
+  dnsNames: string[];
+
+  /** Reference to the cert-manager issuer */
+  issuerRef: {
+    name: string;
+    kind?: string; // ClusterIssuer or Issuer
+  };
+
+  /** Optional duration (default: cert-manager default) */
+  duration?: string;
+
+  /** Optional renewBefore (default: cert-manager default) */
+  renewBefore?: string;
+}
+
+class Certificate extends Construct {
+  public readonly manifest: Manifest;
+
+  constructor(scope: Construct, id: string, opts: CertificateOptions) {
+    super(scope, id);
+
+    const manifest: any = {
+      apiVersion: "cert-manager.io/v1",
+      kind: "Certificate",
+      metadata: {
+        name: opts.name,
+        namespace: opts.namespace,
+      },
+      spec: {
+        secretName: opts.secretName,
+        dnsNames: opts.dnsNames,
+        issuerRef: {
+          name: opts.issuerRef.name,
+          kind: opts.issuerRef.kind ?? "ClusterIssuer",
+        },
+      },
+    };
+
+    if (opts.duration) {
+      manifest.spec.duration = opts.duration;
+    }
+
+    if (opts.renewBefore) {
+      manifest.spec.renewBefore = opts.renewBefore;
+    }
+
+    this.manifest = new Manifest(this, id, {
+      provider: opts.provider,
+      manifest,
+    });
+  }
+}
+
+export class CloudflareCertificate extends Certificate {
+  constructor(
+    scope: Construct,
+    id: string,
+    opts: Omit<CertificateOptions, "issuerRef">,
+  ) {
+    super(scope, id, {
+      ...opts,
+      issuerRef: {
+        name: "cloudflare-issuer",
+        kind: "ClusterIssuer",
+      },
+    });
+  }
+}

cert-manager/index.ts

@@ -5,6 +5,8 @@ import { Construct } from "constructs";
 import { KubernetesProvider } from "@cdktf/provider-kubernetes/lib/provider";
 import { Manifest } from "@cdktf/provider-kubernetes/lib/manifest";
 
+export { CloudflareCertificate } from "./certificate";
+
 type CertManagerOptions = {
   providers: {
     kubernetes: KubernetesProvider;

external-dns/index.ts

@@ -1,27 +0,0 @@
-import * as fs from "fs";
-import { HelmProvider } from "@cdktf/provider-helm/lib/provider";
-import { Release } from "@cdktf/provider-helm/lib/release";
-import { Construct } from "constructs";
-
-type ExternalDNSOptions = {
-  provider: HelmProvider;
-  name: string;
-  namespace: string;
-};
-
-export class ExternalDNS extends Construct {
-  constructor(scope: Construct, id: string, options: ExternalDNSOptions) {
-    super(scope, id);
-
-    new Release(this, "external-dns", {
-      ...options,
-      repository: "oci://registry-1.docker.io/bitnamicharts/",
-      chart: "external-dns",
-      values: [
-        fs.readFileSync("helm/values/externaldns.values.yaml", {
-          encoding: "utf8",
-        }),
-      ],
-    });
-  }
-}

helm/values/externaldns.values.yaml

@@ -1,41 +0,0 @@
-global:
-  security:
-    allowInsecureImages: true # needed for non-official images
-image:
-  registry: docker.io
-  repository: bitnamilegacy/external-dns
-  tag: 0.18.0-debian-12-r1
-  pullPolicy: IfNotPresent
-interval: 10s
-provider: pihole
-policy: upsert-only
-txtOwnerId: "homelab"
-pihole:
-  server: http://rashid
-nodeSelector:
-  nodepool: worker
-extraEnvVars:
-  - name: EXTERNAL_DNS_PIHOLE_PASSWORD
-    valueFrom:
-      secretKeyRef:
-        name: pihole-admin
-        key: app-password
-extraArgs:
-  pihole-api-version: 6
-serviceAccount:
-  create: true
-  name: "external-dns"
-ingressClassFilters:
-  - nginx-internal
-  - traefik
-metrics:
-  enabled: false
-  serviceMonitor:
-    enabled: true
-    interval: 30s
-    scrapeTimeout: 10s
-    selector:
-      matchLabels:
-        app.kubernetes.io/name: external-dns
-        app.kubernetes.io/instance: externaldns-pihole
-    port: 7979

helm/values/nginx-internal.values.yaml

@@ -1,57 +0,0 @@
-controller:
-  replicaCount: 3
-  nodeSelector:
-    nodepool: worker
-  labels:
-    app: nginx-internal
-  topologySpreadConstraints:
-    - maxSkew: 1
-      topologyKey: kubernetes.io/hostname
-      whenUnsatisfiable: DoNotSchedule
-      labelSelector:
-        matchLabels:
-          app: nginx-internal
-  ingressClassResource:
-    name: nginx-internal
-    enabled: true
-    default: true
-    controllerValue: "k8s.io/ingress-nginx"
-    parameters: {}
-  ingressClass: nginx-internal
-  service:
-    annotations:
-      external-dns.alpha.kubernetes.io/hostname: "dogar.dev"
-  extraVolumes:
-    - name: nix-cache
-      persistentVolumeClaim:
-        claimName: nix-cache
-  extraVolumeMounts:
-    - name: nix-cache
-      mountPath: /var/cache/nginx/nix
-  podSecurityContext:
-    fsGroup: 101
-  config:
-    proxy-buffering: "on"
-    proxy-ssl-server-name: "true"
-    http-snippet: |
-      # Persistent on-disk cache; lives on the PVC
-      proxy_cache_path /var/cache/nginx/nix levels=1:2 keys_zone=cachecache:32m max_size=120g inactive=365d use_temp_path=off;
-
-      # Only advertise cacheability for 200/302
-      map $status $cache_header {
-        200     "public";
-        302     "public";
-        default "no-cache";
-      }
-    server-snippet: |
-      location = /robots.txt {
-        default_type text/plain;
-        return 200 "User-agent: GPTBot\nDisallow: /\nUser-agent: CCBot\nDisallow: /\nUser-agent: *\nAllow: /\n";
-      }
-tcp:
-  22:       "homelab/gitea-ssh:22"
-  25565:    "minecraft/monifactory-server:25565"
-  25566:    "minecraft/gtnh-server:25565"
-  25567:    "minecraft/tfg-server:25565"
-  25568:    "minecraft/atm10-server:25565"
-  25569:    "minecraft/star-technology-server:25565"

main.ts

@@ -1,7 +1,7 @@
 import * as dotenv from "dotenv";
 import { cleanEnv, str } from "envalid";
 import { Construct } from "constructs";
-import { App, TerraformStack, LocalBackend } from "cdktf";
+import { App, TerraformStack, LocalBackend, PgBackend } from "cdktf";
 import { HelmProvider } from "@cdktf/provider-helm/lib/provider";
 import { KubernetesProvider } from "@cdktf/provider-kubernetes/lib/provider";
 import { NamespaceV1 } from "@cdktf/provider-kubernetes/lib/namespace-v1";
@@ -13,16 +13,18 @@ import { Longhorn } from "./longhorn";
 import { AuthentikServer } from "./authentik";
 import { ValkeyCluster } from "./valkey";
 import { CertManager } from "./cert-manager";
-import { Nginx } from "./nginx";
 import { Traefik } from "./traefik";
 import { Prometheus } from "./prometheus";
 import { MetalLB } from "./metallb";
-import { ExternalDNS } from "./external-dns";
+import { NixCache } from "./nixcache";
 
 dotenv.config();
 
 const env = cleanEnv(process.env, {
   ACCOUNT_ID: str({ desc: "Cloudflare account id." }),
+  PG_CONN_STR: str({
+    desc: "PostgreSQL connection string for Terraform state backend.",
+  }),
 });
 
 const r2Endpoint = `https://${env.ACCOUNT_ID}.r2.cloudflarestorage.com`;
@@ -69,12 +71,6 @@ class Homelab extends TerraformStack {
       namespace,
     });
 
-    const nginx = new Nginx(this, "nginx", {
-      provider: helm,
-      namespace,
-      name: "nginx-ingress",
-    });
-
     new Traefik(this, "traefik", {
       provider: helm,
       namespace,
@@ -83,7 +79,7 @@ class Homelab extends TerraformStack {
 
     const certManagerApiVersion = "cert-manager.io/v1";
 
-    const cm = new CertManager(this, "cert-manager", {
+    new CertManager(this, "cert-manager", {
       certManagerApiVersion,
       name: "cert-manager",
       namespace,
@@ -94,15 +90,6 @@ class Homelab extends TerraformStack {
       },
     });
 
-    const externalDNS = new ExternalDNS(this, "external-dns", {
-      namespace,
-      provider: helm,
-      name: "external-dns",
-    });
-
-    externalDNS.node.addDependency(nginx);
-    externalDNS.node.addDependency(cm);
-
     new Prometheus(this, "prometheus", {
       provider: helm,
       namespace,
@@ -154,11 +141,19 @@ class Homelab extends TerraformStack {
 }
 
 const app = new App();
-const stack = new Homelab(app, "homelab");
+const homelab = new Homelab(app, "homelab");
 
-new LocalBackend(stack, {
+const nixCache = new NixCache(app, "nix-cache");
+nixCache.node.addDependency(homelab);
+
+new LocalBackend(homelab, {
   path: "terraform.tfstate",
   workspaceDir: ".",
 });
 
+new PgBackend(nixCache, {
+  schemaName: "nix_cache",
+  connStr: env.PG_CONN_STR,
+});
+
 app.synth();

nginx/index.ts

@@ -1,36 +0,0 @@
-import * as fs from "fs";
-import { HelmProvider } from "@cdktf/provider-helm/lib/provider";
-import { Release } from "@cdktf/provider-helm/lib/release";
-import { Construct } from "constructs";
-
-import { NixCache } from "./nix-cache";
-
-type NginxOptions = {
-  provider: HelmProvider;
-  name: string;
-  namespace: string;
-};
-
-export class Nginx extends Construct {
-  constructor(scope: Construct, id: string, options: NginxOptions) {
-    super(scope, id);
-
-    new Release(this, id, {
-      ...options,
-      repository: "https://kubernetes.github.io/ingress-nginx",
-      chart: "ingress-nginx",
-      createNamespace: true,
-      values: [
-        fs.readFileSync("helm/values/nginx-internal.values.yaml", {
-          encoding: "utf8",
-        }),
-      ],
-    });
-
-    new NixCache(this, "nix-cache", {
-      namespace: options.namespace,
-      host: "nix.dogar.dev",
-      ingressClassName: "nginx-internal",
-    });
-  }
-}

nginx/nix-cache.ts

@@ -1,105 +0,0 @@
-import { Construct } from "constructs";
-import { ServiceV1 } from "@cdktf/provider-kubernetes/lib/service-v1";
-import { IngressV1 } from "@cdktf/provider-kubernetes/lib/ingress-v1";
-import { PersistentVolumeClaimV1 } from "@cdktf/provider-kubernetes/lib/persistent-volume-claim-v1";
-
-export interface NixCacheProps {
-  namespace: string;
-  host: string;
-  ingressClassName?: string;
-  externalName?: string;
-}
-
-export class NixCache extends Construct {
-  constructor(scope: Construct, id: string, props: NixCacheProps) {
-    super(scope, id);
-
-    const {
-      namespace,
-      host,
-      ingressClassName: ingressClass = "nginx-internal",
-      externalName: upstreamHost = "cache.nixos.org",
-    } = props;
-
-    // 1) ExternalName Service -> cache.nixos.org
-    new ServiceV1(this, "nixcache-upstream-svc", {
-      metadata: {
-        name: "nixcache-upstream",
-        namespace,
-      },
-      spec: {
-        type: "ExternalName",
-        externalName: upstreamHost,
-      },
-    });
-
-    // 2) Ingress that targets the ExternalName Service over HTTPS:443
-    new IngressV1(this, "nixcache-ingress", {
-      metadata: {
-        name: "nix-cache",
-        namespace,
-        annotations: {
-          // Use the cache zone defined in controller.config.http-snippet
-          "nginx.ingress.kubernetes.io/proxy-cache": "cachecache",
-          "nginx.ingress.kubernetes.io/proxy-cache-valid": "200 302 60d",
-          "nginx.ingress.kubernetes.io/proxy-cache-lock": "true",
-          "nginx.ingress.kubernetes.io/proxy-buffering": "on",
-
-          // Upstream is HTTPS with SNI and a fixed Host header
-          "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
-          "nginx.ingress.kubernetes.io/proxy-ssl-server-name": "true",
-          "nginx.ingress.kubernetes.io/upstream-vhost": upstreamHost,
-
-          // Use cert-manager to provision TLS certs via Cloudflare
-          "cert-manager.io/cluster-issuer": "cloudflare-issuer",
-          "cert-manager.io/acme-challenge-type": "dns01",
-          "cert-manager.io/private-key-size": "4096",
-        },
-      },
-      spec: {
-        ingressClassName: ingressClass,
-        rule: [
-          {
-            host,
-            http: {
-              path: [
-                {
-                  path: "/",
-                  pathType: "Prefix",
-                  backend: {
-                    service: {
-                      name: "nixcache-upstream",
-                      port: { number: 443 },
-                    },
-                  },
-                },
-              ],
-            },
-          },
-        ],
-        tls: [
-          {
-            hosts: [host],
-            secretName: "nix-cache-tls",
-          },
-        ],
-      },
-    });
-
-    // 3) PersistentVolumeClaim for caching
-    new PersistentVolumeClaimV1(this, "nix-cache-pvc", {
-      metadata: {
-        name: "nix-cache",
-        namespace,
-      },
-      spec: {
-        accessModes: ["ReadWriteMany"],
-        resources: {
-          requests: {
-            storage: "128Gi",
-          },
-        },
-      },
-    });
-  }
-}

nixcache/index.ts

@@ -0,0 +1,147 @@
+import * as fs from "fs";
+import * as path from "path";
+import { Construct } from "constructs";
+import { TerraformStack } from "cdktf";
+import { PersistentVolumeClaimV1 } from "@cdktf/provider-kubernetes/lib/persistent-volume-claim-v1";
+import { ConfigMapV1 } from "@cdktf/provider-kubernetes/lib/config-map-v1";
+
+import { DeploymentV1 } from "@cdktf/provider-kubernetes/lib/deployment-v1";
+import { KubernetesProvider } from "@cdktf/provider-kubernetes/lib/provider";
+import { TraefikIngressRoute } from "../traefik/ingress-route";
+import { ServiceV1 } from "@cdktf/provider-kubernetes/lib/service-v1";
+
+export class NixCache extends TerraformStack {
+  constructor(scope: Construct, id: string) {
+    super(scope, id);
+
+    const kubernetes = new KubernetesProvider(this, "kubernetes", {
+      configPath: "~/.kube/config",
+    });
+
+    const pvc = new PersistentVolumeClaimV1(this, "pvc", {
+      provider: kubernetes,
+      metadata: {
+        name: "nix-cache",
+        namespace: "homelab",
+      },
+      spec: {
+        storageClassName: "longhorn",
+        accessModes: ["ReadWriteMany"],
+        resources: {
+          requests: {
+            storage: "64Gi",
+          },
+        },
+      },
+    });
+
+    const nginxConfig = fs.readFileSync(
+      path.join(__dirname, "./nginx.conf"),
+      "utf-8",
+    );
+
+    const configMap = new ConfigMapV1(this, "config-map", {
+      provider: kubernetes,
+      metadata: {
+        name: "nix-cache",
+        namespace: "homelab",
+      },
+      data: {
+        "nix-cache.conf": nginxConfig,
+      },
+    });
+
+    new ServiceV1(this, "service", {
+      provider: kubernetes,
+      metadata: {
+        name: "nix-cache",
+        namespace: "homelab",
+      },
+      spec: {
+        selector: {
+          app: "nix-cache",
+        },
+        port: [
+          {
+            name: "http",
+            port: 80,
+            targetPort: "80",
+          },
+        ],
+        type: "ClusterIP",
+      },
+    });
+
+    new DeploymentV1(this, "deployment", {
+      provider: kubernetes,
+      metadata: {
+        name: "nix-cache",
+        namespace: "homelab",
+      },
+      spec: {
+        replicas: "3",
+        selector: {
+          matchLabels: {
+            app: "nix-cache",
+          },
+        },
+        template: {
+          metadata: {
+            labels: {
+              app: "nix-cache",
+            },
+          },
+          spec: {
+            container: [
+              {
+                name: "nginx",
+                image: "nginx:latest",
+                volumeMount: [
+                  {
+                    name: "cache",
+                    mountPath: "/var/cache/nginx/nix",
+                  },
+                  {
+                    name: "nginx-config",
+                    mountPath: "/etc/nginx/conf.d/nix-cache.conf",
+                    subPath: "nix-cache.conf",
+                  },
+                ],
+              },
+            ],
+            volume: [
+              {
+                name: "cache",
+                persistentVolumeClaim: {
+                  claimName: pvc.metadata.name,
+                },
+              },
+              {
+                name: "nginx-config",
+                configMap: {
+                  name: configMap.metadata.name,
+                  items: [
+                    {
+                      key: "nix-cache.conf",
+                      path: "nix-cache.conf",
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        },
+      },
+    });
+
+    new TraefikIngressRoute(this, "ingress-route", {
+      provider: kubernetes,
+      namespace: "homelab",
+      host: "nix.dogar.dev",
+      serviceName: "nix-cache",
+      servicePort: 80,
+      entryPoints: ["websecure"],
+      tlsSecretName: "nix-cache-tls",
+    });
+  }
+}

nixcache/nginx.conf

@@ -0,0 +1,32 @@
+proxy_cache_path /var/cache/nginx/nix levels=1:2 keys_zone=nix_cache:32m max_size=60g inactive=365d use_temp_path=off;
+
+map $status $cache_header {
+  200     "public";
+  302     "public";
+  default "no-cache";
+}
+
+server {
+  listen 80;
+  server_name nix.dogar.dev;
+
+  location / {
+    proxy_pass https://cache.nixos.org;
+    proxy_http_version 1.1;
+
+    proxy_set_header Host cache.nixos.org;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    proxy_ssl_server_name on;
+
+    proxy_cache nix_cache;
+    proxy_cache_valid 200 302 365d;
+    proxy_cache_valid any 1m;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+
+    add_header X-Cache-Status $upstream_cache_status always;
+    add_header Cache-Control $cache_header always;
+  }
+}

traefik/index.ts

@@ -1,4 +1,5 @@
 import * as fs from "fs";
+import * as path from "path";
 import { HelmProvider } from "@cdktf/provider-helm/lib/provider";
 import { Release } from "@cdktf/provider-helm/lib/release";
 import { Construct } from "constructs";
@@ -19,7 +20,7 @@ export class Traefik extends Construct {
       chart: "traefik",
       createNamespace: true,
       values: [
-        fs.readFileSync("helm/values/traefik.values.yaml", {
+        fs.readFileSync(path.join(__dirname, "values.yaml"), {
           encoding: "utf8",
         }),
       ],

traefik/ingress-route.ts

@@ -0,0 +1,94 @@
+import { Construct } from "constructs";
+import { Manifest } from "@cdktf/provider-kubernetes/lib/manifest";
+import { KubernetesProvider } from "@cdktf/provider-kubernetes/lib/provider";
+
+import { CloudflareCertificate } from "../cert-manager";
+
+export interface TraefikIngressRouteOptions {
+  provider: KubernetesProvider;
+  namespace: string;
+
+  /** Hostname for this route (e.g. npm.dogar.dev) */
+  host: string;
+
+  /** Path prefix (default: "/") */
+  path?: string;
+
+  /** Backend K8s Service */
+  serviceName: string;
+  servicePort: number;
+
+  /** EntryPoints (default: ["websecure"]) */
+  entryPoints?: string[];
+
+  /** TLS secret name for HTTPS termination */
+  tlsSecretName?: string;
+
+  /** Extra middlewares (traefik format: namespace/name) */
+  middlewares?: string[];
+
+  /** Name override (otherwise auto) */
+  name?: string;
+}
+
+export class TraefikIngressRoute extends Construct {
+  public readonly manifest: Manifest;
+
+  constructor(scope: Construct, id: string, opts: TraefikIngressRouteOptions) {
+    super(scope, id);
+
+    const name = opts.name ?? `route-${opts.host.replace(/\./g, "-")}`;
+    const path = opts.path ?? "/";
+    const entryPoints = opts.entryPoints ?? ["websecure"];
+
+    const route: any = {
+      match: `Host(\`${opts.host}\`) && PathPrefix(\`${path}\`)`,
+      kind: "Rule",
+      services: [
+        {
+          name: opts.serviceName,
+          port: opts.servicePort,
+        },
+      ],
+    };
+
+    if (opts.middlewares?.length) {
+      route.middlewares = opts.middlewares.map((mw) => {
+        const [namespace, name] = mw.split("/");
+        return { name, namespace };
+      });
+    }
+
+    const spec: any = {
+      entryPoints,
+      routes: [route],
+    };
+
+    if (opts.tlsSecretName) {
+      spec.tls = {
+        secretName: opts.tlsSecretName,
+      };
+
+      new CloudflareCertificate(this, `${name}-cert`, {
+        provider: opts.provider,
+        namespace: opts.namespace,
+        name: opts.host,
+        secretName: opts.tlsSecretName,
+        dnsNames: [opts.host],
+      });
+    }
+
+    this.manifest = new Manifest(this, name, {
+      provider: opts.provider,
+      manifest: {
+        apiVersion: "traefik.io/v1alpha1",
+        kind: "IngressRoute",
+        metadata: {
+          name,
+          namespace: opts.namespace,
+        },
+        spec,
+      },
+    });
+  }
+}