apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: rabbitmq-data, namespace: enkly} spec: accessModes: [ReadWriteOnce] storageClassName: managed-nfs-storage resources: {requests: {storage: 10Gi}} --- # Root cause of the alarm in the pod logs: RabbitMQ's default memory # watermark is 40% of what it detects as "total system memory". Inside # this container that detection comes back as the *node's* memory # (7949 MiB), not the 512Mi the container is actually capped to — so # the watermark it set (3179 MiB) is six times higher than the limit # Kubernetes will actually enforce. In practice that means the broker # never throttles publishers in time and gets OOMKilled by the kubelet # instead of gracefully blocking connections, which is what the # watermark is supposed to prevent. Pinning the watermark explicitly, # below the container's real memory limit, fixes that. apiVersion: v1 kind: ConfigMap metadata: {name: rabbitmq-conf, namespace: enkly} data: 20-enkly.conf: | # Keep this comfortably under the container memory limit below so # RabbitMQ blocks publishers well before Kubernetes OOMKills the pod. vm_memory_high_watermark.absolute = 550MB vm_memory_calculation_strategy = allocated disk_free_limit.absolute = 2GB # NOTE: management_metrics_collection logs its deprecation warning on # every boot regardless of the deprecated_features.permit setting while # it's in the "permitted by default" phase — there is no config to # silence it. It's informational, not actionable, until RabbitMQ moves # it to "denied by default" in a future release. The real work item is # migrating anything scraping the management API for metrics (Grafana, # alerting) over to the rabbitmq_prometheus plugin before that happens. --- apiVersion: apps/v1 kind: Deployment metadata: name: rabbitmq namespace: enkly labels: {app: rabbitmq} spec: replicas: 1 strategy: {type: Recreate} selector: {matchLabels: {app: rabbitmq}} template: metadata: labels: {app: rabbitmq} spec: imagePullSecrets: [{name: enkly-registry}] # RabbitMQ resolves hostnames during boot even with RABBITMQ_NODENAME # pinned to rabbit@localhost (the management plugin and Erlang's # networking stack still do their own lookups). The default resolver # in a pod walks every *.svc.cluster.local search-domain suffix # before giving up on each failed query, and RabbitMQ's own k8s # deployment docs call out exactly this as a cause of ~30-50s boot # stalls. Shortening the per-query timeout/attempts here is the # pod-level fix that doesn't require touching cluster-wide CoreDNS. dnsPolicy: ClusterFirst dnsConfig: options: - {name: ndots, value: "2"} - {name: timeout, value: "2"} - {name: attempts, value: "2"} # Runs before the rabbitmq container every single boot, sharing the # same PVC. Deletes any leftover mnesia directory that isn't the one # actually in use (rabbit@localhost). Those directories are what # caused the ~120s boot time and the resulting liveness-probe kills: # every pod restart before RABBITMQ_NODENAME was pinned minted a new # directory named after that pod's hostname, and the entrypoint's # ownership pass over /var/lib/rabbitmq scans all of them. This is # idempotent — once clean, it's a no-op on every future boot, and it # means boot time is never again at the mercy of accumulated cruft, # so the probe timings below don't need to keep being widened. initContainers: - name: clean-stale-mnesia-dirs image: rabbitmq:3.13-management-alpine command: - sh - -c - | set -e cd /var/lib/rabbitmq/mnesia 2>/dev/null || exit 0 for d in rabbit@*; do [ -e "$d" ] || continue case "$d" in rabbit@localhost|rabbit@localhost-plugins-expand|rabbit@localhost-feature_flags) continue ;; esac echo "removing stale mnesia dir: $d" rm -rf -- "$d" done volumeMounts: - {name: data, mountPath: /var/lib/rabbitmq} containers: - name: rabbitmq image: rabbitmq:3.13-management-alpine ports: - {name: amqp, containerPort: 5672} - {name: mgmt, containerPort: 15672} env: # Pins the Erlang node identity so it does NOT follow the pod's # hostname. Without this, every rollout/reschedule gets a new # pod name -> new node name (rabbit@) -> RabbitMQ # mounts the same PVC but looks for mnesia data under a path # for a node it's never seen before, so it reports "virgin # node" and silently reseeds an empty vhost every time. Safe # as a static value here because replicas is 1 and strategy is # Recreate, so there's never two pods needing distinct identities. - name: RABBITMQ_NODENAME value: "rabbit@localhost" - name: RABBITMQ_DEFAULT_USER valueFrom: {configMapKeyRef: {name: enkly-config, key: RABBITMQ_DEFAULT_USER}} - name: RABBITMQ_DEFAULT_PASS valueFrom: {secretKeyRef: {name: enkly-secrets, key: RABBITMQ_DEFAULT_PASS}} resources: # request == limit (Guaranteed QoS): a Burstable broker pod is # one of the kubelet's first eviction targets under node memory # pressure, which is the last thing you want for the message # bus. 768Mi gives the 550MB watermark above real headroom # before the container limit, instead of the old 512Mi limit # sitting below the watermark that was actually configured. # Raised from 250m/500m: gives the liveness probe's # rabbitmq-diagnostics CLI (a separate, short-lived BEAM VM # each time it runs) enough CPU headroom to finish inside its # 5s timeout without contending with the broker process itself. requests: {cpu: 500m, memory: 768Mi} limits: {cpu: 500m, memory: 768Mi} readinessProbe: # Switched from an exec-based "rabbitmq-diagnostics # check_port_connectivity" to a plain TCP check on the AMQP # port. The exec form spins up a brand-new Erlang/BEAM VM on # every single invocation just to run the check — under this # container's CPU limit that overhead can exceed the probe's # own timeout even while the broker itself is completely # healthy, which reads as a permanently-failing readiness # probe with no corresponding error in the app's own logs. # RabbitMQ's own Kubernetes deployment docs recommend a # TCP-based readiness check for exactly this reason — it's # handled by the kubelet directly with no process spawned # inside the container, so it isn't subject to CPU contention # with the broker or the probe's own VM startup cost. tcpSocket: {port: amqp} initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 10 livenessProbe: exec: {command: ["rabbitmq-diagnostics", "-q", "ping"]} initialDelaySeconds: 30 periodSeconds: 20 timeoutSeconds: 5 failureThreshold: 6 volumeMounts: - {name: data, mountPath: /var/lib/rabbitmq} - {name: conf, mountPath: /etc/rabbitmq/conf.d/20-enkly.conf, subPath: 20-enkly.conf} volumes: - name: data persistentVolumeClaim: {claimName: rabbitmq-data} - name: conf configMap: {name: rabbitmq-conf} --- apiVersion: v1 kind: Service metadata: name: rabbitmq namespace: enkly labels: {app: rabbitmq} spec: selector: {app: rabbitmq} ports: - {name: amqp, port: 5672, targetPort: amqp} - {name: mgmt, port: 15672, targetPort: mgmt}