K3s and RKE2

kubectl -n scafo get configmap scafo-node -o yaml
kubectl -n scafo get secret scafo-bucket -o jsonpath='{.data.SCAFO_S3_ACCESS_KEY}' | base64 -d; echo
kubectl -n scafo get secret scafo-bucket -o jsonpath='{.data.SCAFO_S3_SECRET_KEY}' | base64 -d; echo

scafo remote add prod --endpoint https://apps.example.com --register
scafo deploy ./my-app --name shop

apiVersion: v1
kind: ConfigMap
metadata:
  name: scafo-node
  namespace: scafo
data:
  # These non-secret defaults identify one fleet; set them for the external store.
  SCAFO_S3_ENDPOINT: "https://s3.example.com"
  SCAFO_S3_BUCKET: "scafo"
  SCAFO_S3_REGION: "us-east-1"
  SCAFO_S3_PREFIX: "production"
  SCAFO_PLATFORM_DOMAINS: "apps.example.com"
  SCAFO_LISTEN: "0.0.0.0:7400"
  # `none` declares the pod network private: runtime processes bind the pod
  # address, so the cluster's Traefik dials whichever pod holds a host directly.
  SCAFO_TRANSPORT: "none"
  # The entryPoint the routers are attached to is the TLS one of the bundled
  # Traefik; see traefik-config.yaml, which is what polls this routing.
  SCAFO_TRAEFIK_ENTRYPOINTS: "websecure"
  SCAFO_OBSERVABILITY: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: scafo-nodes
  namespace: scafo
spec:
  clusterIP: None
  selector:
    app.kubernetes.io/name: scafo
  ports:
    - name: api
      port: 7400
      targetPort: api
---
# Where the bundled Traefik polls the fleet's routing. Every node renders the
# same document, so this Service may answer from any of them; it asks for no
# preferred node, which spreads each host over every pod serving it.
apiVersion: v1
kind: Service
metadata:
  name: scafo-api
  namespace: scafo
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: scafo
  ports:
    - name: api
      port: 7400
      targetPort: api
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: scafo-node
  namespace: scafo
  labels:
    app.kubernetes.io/name: scafo
spec:
  # One node to start with. Scaling out is supported: the cluster edge reads the
  # whole fleet's routing, so it reaches a host on whichever pod holds it.
  replicas: 1
  serviceName: scafo-nodes
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Retain
  selector:
    matchLabels:
      app.kubernetes.io/name: scafo
  template:
    metadata:
      labels:
        app.kubernetes.io/name: scafo
    spec:
      # Keep node and descendants inside the container lifecycle.
      hostPID: false
      shareProcessNamespace: false
      # Celld shutdown may take 40 seconds; permit Scafo to drain first.
      terminationGracePeriodSeconds: 50
      securityContext:
        fsGroup: 10001
      containers:
        - name: node
          # Replace VERSION with a published Scafo release before applying.
          image: ghcr.io/sandros94/scafo:VERSION
          imagePullPolicy: IfNotPresent
          args: ["sail"]
          securityContext:
            runAsNonRoot: true
            runAsUser: 10001
            runAsGroup: 10001
            allowPrivilegeEscalation: false
          envFrom:
            # Create these Secrets outside the manifest: `scafo-bucket` holds the
            # bucket-root credentials, `scafo-ingress` the SCAFO_INGRESS_TOKEN the
            # cluster's Traefik presents to read the fleet's routing. That token
            # needs a second copy in `kube-system`, where Traefik runs.
            - secretRef:
                name: scafo-bucket
            - secretRef:
                name: scafo-ingress
            - configMapRef:
                name: scafo-node
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: SCAFO_ADVERTISE
              value: "$(POD_IP):7400"
            - name: SCAFO_NODE_LABELS
              value: "platform=k3s"
          ports:
            - name: api
              containerPort: 7400
          volumeMounts:
            - name: state
              mountPath: /var/lib/scafo
          startupProbe:
            httpGet:
              path: /.well-known/scafo/health
              port: api
            periodSeconds: 5
            failureThreshold: 24
          readinessProbe:
            httpGet:
              path: /.well-known/scafo/health
              port: api
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /.well-known/scafo/health
              port: api
            periodSeconds: 20
            failureThreshold: 3
          resources:
            # Cgroup bounds do not influence Scafo placement or deploy admission.
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              memory: "2Gi"
  # A claim per pod, kept when the StatefulSet is scaled down or deleted, and
  # never an emptyDir: each runtime process keeps its work directory here, a
  # write is acknowledged once a peer holds it there before the bucket has it,
  # and the pod recovers it at boot from what its predecessor of the same name
  # left. K3s' default StorageClass is node-local, so a pod whose node is gone
  # stays Pending until that node is back rather than restarting empty.
  volumeClaimTemplates:
    - metadata:
        name: state
      spec:
        # Omit storageClassName to use the K3s default StorageClass.
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: "1Gi"
---
# cert-manager and this ClusterIssuer are operator-managed prerequisites.
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: scafo-platform
  namespace: scafo
spec:
  secretName: scafo-platform-tls
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
    group: cert-manager.io
  dnsNames:
    - apps.example.com
    - "*.apps.example.com"
---
# The routing comes from Scafo's http provider, so nothing here routes anything.
# This Ingress exists to hand the wildcard Secret to Traefik: its Kubernetes
# provider reads `spec.tls` into the certificate store before it looks at rules,
# so an Ingress with no rules adds the certificate and creates no router. The
# store is matched by SNI, which is how the platform hosts of the `@http`
# routers are served this certificate rather than an ACME-issued one.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: scafo-platform-tls
  namespace: scafo
spec:
  ingressClassName: traefik
  tls:
    - hosts:
        - apps.example.com
        - "*.apps.example.com"
      secretName: scafo-platform-tls

# Customises K3s's bundled Traefik so it reads the fleet's routing itself.
#
# Copy this file to /var/lib/rancher/k3s/server/manifests/traefik-config.yaml on
# every server node; the filename must be RFC-1123. K3s's helm-controller merges
# it into the bundled chart and upgrades the release. Never edit the neighbouring
# traefik.yaml: K3s rewrites it on every start. On RKE2 the directory is
# /var/lib/rancher/rke2/server/manifests/ and the chart is named `rke2-traefik`.
#
# Requires K3s or RKE2 v1.32 or newer, whose chart ships Traefik v3: the bearer
# header below is `providers.http.headers`, which Traefik 2.11 does not have.
apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
  name: traefik
  namespace: kube-system
spec:
  valuesContent: |-
    # A Traefik provider is static configuration, settled at startup, and the
    # chart has no value for the http one — its schema rejects unknown keys — so
    # arguments are the only way to declare it.
    #
    # Kubernetes expands $(VAR) in a container's arguments from that container's
    # environment, so the token stays in the Secret rather than in this file.
    additionalArguments:
      - "--providers.http.endpoint=http://scafo-api.scafo.svc:7400/v1/ingress/routes/traefik"
      - "--providers.http.pollInterval=5s"
      - "--providers.http.headers.Authorization=Bearer $(SCAFO_INGRESS_TOKEN)"
    # Traefik runs in kube-system, so the ingress token needs a copy there. It is
    # the only credential Traefik holds; it never sees bucket credentials.
    env:
      - name: SCAFO_INGRESS_TOKEN
        valueFrom:
          secretKeyRef:
            name: scafo-ingress
            key: SCAFO_INGRESS_TOKEN
    # Scafo's routers are attached to `websecure` (SCAFO_TRAEFIK_ENTRYPOINTS);
    # `asDefault` keeps operator-owned Kubernetes routes on the same entryPoint.
    ports:
      websecure:
        asDefault: true
        http:
          tls:
            enabled: true
            certResolver: letsencrypt
    # For domains an application claimed: their DNS points at this cluster, so
    # HTTP-01 on the `web` entryPoint resolves without a DNS-provider plugin.
    # The platform apex and its wildcard are not issued here — cert-manager holds
    # them, and ACME skips a host the certificate store already covers.
    certificatesResolvers:
      letsencrypt:
        acme:
          email: "operator@example.com"
          storage: /data/acme.json
          httpChallenge:
            entryPoint: web
    # Issued certificates survive a restart only on a volume. The chart mounts
    # the claim at /data as uid 65532; acme.json must exist with mode 600 before
    # Traefik reads it, which is what the init container is for.
    persistence:
      enabled: true
      size: 128Mi
    deployment:
      initContainers:
        - name: volume-permissions
          image: busybox:latest
          command: ["sh", "-c", "touch /data/acme.json; chmod -v 600 /data/acme.json"]
          volumeMounts:
            - name: data
              mountPath: /data
    podSecurityContext:
      fsGroup: 65532