174 lines
8.0 KiB
YAML
174 lines
8.0 KiB
YAML
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: rabbitmq-data, namespace: enkly}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: managed-nfs-storage
|
|
resources: {requests: {storage: 10Gi}}
|
|
---
|
|
# Root cause of the alarm in the pod logs: RabbitMQ's default memory
|
|
# watermark is 40% of what it detects as "total system memory". Inside
|
|
# this container that detection comes back as the *node's* memory
|
|
# (7949 MiB), not the 512Mi the container is actually capped to — so
|
|
# the watermark it set (3179 MiB) is six times higher than the limit
|
|
# Kubernetes will actually enforce. In practice that means the broker
|
|
# never throttles publishers in time and gets OOMKilled by the kubelet
|
|
# instead of gracefully blocking connections, which is what the
|
|
# watermark is supposed to prevent. Pinning the watermark explicitly,
|
|
# below the container's real memory limit, fixes that.
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: rabbitmq-conf, namespace: enkly}
|
|
data:
|
|
20-enkly.conf: |
|
|
# Keep this comfortably under the container memory limit below so
|
|
# RabbitMQ blocks publishers well before Kubernetes OOMKills the pod.
|
|
vm_memory_high_watermark.absolute = 550MB
|
|
vm_memory_calculation_strategy = allocated
|
|
disk_free_limit.absolute = 2GB
|
|
# NOTE: management_metrics_collection logs its deprecation warning on
|
|
# every boot regardless of the deprecated_features.permit setting while
|
|
# it's in the "permitted by default" phase — there is no config to
|
|
# silence it. It's informational, not actionable, until RabbitMQ moves
|
|
# it to "denied by default" in a future release. The real work item is
|
|
# migrating anything scraping the management API for metrics (Grafana,
|
|
# alerting) over to the rabbitmq_prometheus plugin before that happens.
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: rabbitmq
|
|
namespace: enkly
|
|
labels: {app: rabbitmq}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: rabbitmq}}
|
|
template:
|
|
metadata:
|
|
labels: {app: rabbitmq}
|
|
spec:
|
|
imagePullSecrets: [{name: enkly-registry}]
|
|
# RabbitMQ resolves hostnames during boot even with RABBITMQ_NODENAME
|
|
# pinned to rabbit@localhost (the management plugin and Erlang's
|
|
# networking stack still do their own lookups). The default resolver
|
|
# in a pod walks every *.svc.cluster.local search-domain suffix
|
|
# before giving up on each failed query, and RabbitMQ's own k8s
|
|
# deployment docs call out exactly this as a cause of ~30-50s boot
|
|
# stalls. Shortening the per-query timeout/attempts here is the
|
|
# pod-level fix that doesn't require touching cluster-wide CoreDNS.
|
|
dnsPolicy: ClusterFirst
|
|
dnsConfig:
|
|
options:
|
|
- {name: ndots, value: "2"}
|
|
- {name: timeout, value: "2"}
|
|
- {name: attempts, value: "2"}
|
|
# Runs before the rabbitmq container every single boot, sharing the
|
|
# same PVC. Deletes any leftover mnesia directory that isn't the one
|
|
# actually in use (rabbit@localhost). Those directories are what
|
|
# caused the ~120s boot time and the resulting liveness-probe kills:
|
|
# every pod restart before RABBITMQ_NODENAME was pinned minted a new
|
|
# directory named after that pod's hostname, and the entrypoint's
|
|
# ownership pass over /var/lib/rabbitmq scans all of them. This is
|
|
# idempotent — once clean, it's a no-op on every future boot, and it
|
|
# means boot time is never again at the mercy of accumulated cruft,
|
|
# so the probe timings below don't need to keep being widened.
|
|
initContainers:
|
|
- name: clean-stale-mnesia-dirs
|
|
image: rabbitmq:3.13-management-alpine
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
cd /var/lib/rabbitmq/mnesia 2>/dev/null || exit 0
|
|
for d in rabbit@*; do
|
|
[ -e "$d" ] || continue
|
|
case "$d" in
|
|
rabbit@localhost|rabbit@localhost-plugins-expand|rabbit@localhost-feature_flags)
|
|
continue ;;
|
|
esac
|
|
echo "removing stale mnesia dir: $d"
|
|
rm -rf -- "$d"
|
|
done
|
|
volumeMounts:
|
|
- {name: data, mountPath: /var/lib/rabbitmq}
|
|
containers:
|
|
- name: rabbitmq
|
|
image: rabbitmq:3.13-management-alpine
|
|
ports:
|
|
- {name: amqp, containerPort: 5672}
|
|
- {name: mgmt, containerPort: 15672}
|
|
env:
|
|
# Pins the Erlang node identity so it does NOT follow the pod's
|
|
# hostname. Without this, every rollout/reschedule gets a new
|
|
# pod name -> new node name (rabbit@<pod-name>) -> RabbitMQ
|
|
# mounts the same PVC but looks for mnesia data under a path
|
|
# for a node it's never seen before, so it reports "virgin
|
|
# node" and silently reseeds an empty vhost every time. Safe
|
|
# as a static value here because replicas is 1 and strategy is
|
|
# Recreate, so there's never two pods needing distinct identities.
|
|
- name: RABBITMQ_NODENAME
|
|
value: "rabbit@localhost"
|
|
- name: RABBITMQ_DEFAULT_USER
|
|
valueFrom: {configMapKeyRef: {name: enkly-config, key: RABBITMQ_DEFAULT_USER}}
|
|
- name: RABBITMQ_DEFAULT_PASS
|
|
valueFrom: {secretKeyRef: {name: enkly-secrets, key: RABBITMQ_DEFAULT_PASS}}
|
|
resources:
|
|
# request == limit (Guaranteed QoS): a Burstable broker pod is
|
|
# one of the kubelet's first eviction targets under node memory
|
|
# pressure, which is the last thing you want for the message
|
|
# bus. 768Mi gives the 550MB watermark above real headroom
|
|
# before the container limit, instead of the old 512Mi limit
|
|
# sitting below the watermark that was actually configured.
|
|
# Raised from 250m/500m: gives the liveness probe's
|
|
# rabbitmq-diagnostics CLI (a separate, short-lived BEAM VM
|
|
# each time it runs) enough CPU headroom to finish inside its
|
|
# 5s timeout without contending with the broker process itself.
|
|
requests: {cpu: 500m, memory: 768Mi}
|
|
limits: {cpu: 500m, memory: 768Mi}
|
|
readinessProbe:
|
|
# Switched from an exec-based "rabbitmq-diagnostics
|
|
# check_port_connectivity" to a plain TCP check on the AMQP
|
|
# port. The exec form spins up a brand-new Erlang/BEAM VM on
|
|
# every single invocation just to run the check — under this
|
|
# container's CPU limit that overhead can exceed the probe's
|
|
# own timeout even while the broker itself is completely
|
|
# healthy, which reads as a permanently-failing readiness
|
|
# probe with no corresponding error in the app's own logs.
|
|
# RabbitMQ's own Kubernetes deployment docs recommend a
|
|
# TCP-based readiness check for exactly this reason — it's
|
|
# handled by the kubelet directly with no process spawned
|
|
# inside the container, so it isn't subject to CPU contention
|
|
# with the broker or the probe's own VM startup cost.
|
|
tcpSocket: {port: amqp}
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 10
|
|
livenessProbe:
|
|
exec: {command: ["rabbitmq-diagnostics", "-q", "ping"]}
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 20
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
volumeMounts:
|
|
- {name: data, mountPath: /var/lib/rabbitmq}
|
|
- {name: conf, mountPath: /etc/rabbitmq/conf.d/20-enkly.conf, subPath: 20-enkly.conf}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim: {claimName: rabbitmq-data}
|
|
- name: conf
|
|
configMap: {name: rabbitmq-conf}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: rabbitmq
|
|
namespace: enkly
|
|
labels: {app: rabbitmq}
|
|
spec:
|
|
selector: {app: rabbitmq}
|
|
ports:
|
|
- {name: amqp, port: 5672, targetPort: amqp}
|
|
- {name: mgmt, port: 15672, targetPort: mgmt} |