327 lines
16 KiB
YAML
327 lines
16 KiB
YAML
# vLLM, per your request — but still on kmaster/kworker1/kworker2,
|
|
# which have no physical GPU. The original vllm-chat/vllm-embed
|
|
# (restored structurally below: two Deployments, one model each, same
|
|
# as pkg/vllmkit's doc comment describes) ran the CUDA vllm/vllm-openai
|
|
# image and requested nvidia.com/gpu: "1" — that's what made both pods
|
|
# sit Pending forever with no node able to advertise that resource.
|
|
#
|
|
# This version runs vLLM's CPU backend instead, built from source
|
|
# with AVX-512 disabled, rather than vLLM's official pre-built
|
|
# public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo image. Two real,
|
|
# observed problems drove both the image choice and the sizing below
|
|
# — not hypothetical ones:
|
|
#
|
|
# 1. vllm-embed's init container (`check-cpu-avx512` further down)
|
|
# reported no `avx512f` CPU flag on the node it landed on. That's
|
|
# consistent with VirtualBox/Vagrant nested virtualization, which
|
|
# historically doesn't expose AVX-512 to guest VMs at all — and
|
|
# this cluster's own vagrant-provisioning/ setup is exactly that.
|
|
# The pre-built image hard-requires avx512f and crashes with
|
|
# "Illegal instruction" without it; there is no runtime flag to
|
|
# turn that off, only a build-time one. So `image:` below now
|
|
# points at a custom build instead — **you must build and push it
|
|
# before applying this file**:
|
|
# git clone https://github.com/vllm-project/vllm.git && cd vllm
|
|
# docker build -f docker/Dockerfile.cpu \
|
|
# --build-arg VLLM_CPU_DISABLE_AVX512=true \
|
|
# --tag registry.enkly.org/enkly/vllm-cpu:latest \
|
|
# --target vllm-openai .
|
|
# docker push registry.enkly.org/enkly/vllm-cpu:latest
|
|
# This build falls back to plain AVX2, which is universal on any
|
|
# x86_64 node (physical or virtualized) built since ~2013 — there's
|
|
# no equivalent flag check needed for it the way avx512f needed
|
|
# one, so the init container that used to gate on avx512f is gone;
|
|
# only a lightweight informational one remains (see below).
|
|
#
|
|
# 2. vllm-chat's 8Gi memory *request* alone exceeded a single node's
|
|
# total capacity: 10-rabbitmq.yaml's memory-watermark fix already
|
|
# established each node has ~7949Mi (~7.76Gi) total RAM, so an 8Gi
|
|
# request could never be satisfied regardless of what else was
|
|
# scheduled — that's what "Pending" (not a crash) meant. Fixed two
|
|
# ways together: the chat model dropped from Qwen2.5-3B-Instruct to
|
|
# Qwen2.5-1.5B-Instruct (bf16 weights ~3GB vs. ~6GB), and
|
|
# resources.requests/limits and VLLM_CPU_KVCACHE_SPACE below were
|
|
# cut to fit comfortably inside a ~7.76Gi node alongside everything
|
|
# else already running there (4Gi/6Gi request/limit — realistic for
|
|
# ~3GB weights + 1GiB KV cache + CPU-backend runtime overhead, not
|
|
# the bare-minimum 3Gi this previously requested, which left too
|
|
# little room to the limit during startup warmup). If
|
|
# `kubectl describe node` shows more real headroom than that once
|
|
# this is stable, both the model and these numbers can be raised
|
|
# again — see the sizing comments inline.
|
|
#
|
|
# 3. The two Deployments previously shared one ReadWriteMany PVC for
|
|
# the Hugging Face cache. managed-nfs-storage (nfs-subdir-external-
|
|
# provisioner) does advertise RWX, so that likely would have bound
|
|
# — but an unverified "likely" is exactly the kind of thing that
|
|
# produces a silent Pending pod (a PVC stuck Pending blocks its
|
|
# Pod the same way insufficient node resources does, just with a
|
|
# different `kubectl describe pod` event, easy to mis-read as a
|
|
# repeat of problem #2). Split into vllm-chat-cache and
|
|
# vllm-embed-cache below, both plain ReadWriteOnce — the same
|
|
# access mode every other PVC in this project already binds with
|
|
# (10-rabbitmq.yaml, 12-databases.yaml, 23-workspace.yaml) — so
|
|
# neither Deployment's scheduling can ever depend on the other's
|
|
# PVC or on RWX support this cluster's storage class may or may
|
|
# not actually have been exercised with yet.
|
|
#
|
|
# Everything else about the original design is unchanged: two
|
|
# Deployments (one vLLM engine process serves exactly one model), the
|
|
# OpenAI-compatible /v1/chat/completions + /v1/embeddings API,
|
|
# --api-key, tool calling — so pkg/vllmkit needs no code changes,
|
|
# only this file and 01-config.yaml's model/URL values.
|
|
#
|
|
# If real GPU nodes get added to this cluster later: point `image:` on
|
|
# both Deployments back at `vllm/vllm-openai`, add back
|
|
# `nvidia.com/gpu` requests + a GPU-pool nodeSelector/toleration + an
|
|
# NVIDIA device-plugin DaemonSet, and size the model/resources up.
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: vllm-chat-cache, namespace: enkly}
|
|
spec:
|
|
# ReadWriteOnce, deliberately not a single shared ReadWriteMany PVC
|
|
# across both Deployments. nfs-subdir-external-provisioner (behind
|
|
# managed-nfs-storage) does advertise RWX, so a shared PVC would
|
|
# probably bind — but "probably" is exactly the kind of unverified
|
|
# assumption that produces a silent, hard-to-diagnose Pending pod
|
|
# (PVC stuck Pending blocks the Pod the same way insufficient node
|
|
# resources does, just with a different `kubectl describe pod`
|
|
# event). Two independent RWO claims remove that failure mode
|
|
# entirely: each is the same well-proven access mode every other
|
|
# PVC in this project already binds with (10-rabbitmq.yaml,
|
|
# 12-databases.yaml, 23-workspace.yaml), neither Deployment's
|
|
# scheduling depends on the other's PVC, and losing the (small,
|
|
# one-time-download) benefit of a shared cache between a 3GB chat
|
|
# model and a 550MB embed model costs nothing worth trading that
|
|
# reliability for.
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: managed-nfs-storage
|
|
# Qwen2.5-1.5B-Instruct at bf16 is ~3GB on disk; sized with headroom
|
|
# to try Qwen2.5-3B-Instruct (~6GB bf16) later without resizing
|
|
# first, once `kubectl describe node` confirms real free RAM.
|
|
resources: {requests: {storage: 10Gi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: vllm-embed-cache, namespace: enkly}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: managed-nfs-storage
|
|
# nomic-embed-text-v1.5 is ~550MB on disk.
|
|
resources: {requests: {storage: 5Gi}}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: vllm-chat, namespace: enkly, labels: {app: vllm-chat}}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: vllm-chat}}
|
|
template:
|
|
metadata: {labels: {app: vllm-chat}}
|
|
spec:
|
|
imagePullSecrets: [{name: enkly-registry}]
|
|
# Deliberately NO nodeSelector/toleration/nvidia.com/gpu request
|
|
# here (unlike the original GPU version of this file) — this
|
|
# cluster has no node that could ever satisfy either.
|
|
initContainers:
|
|
- name: check-cpu-avx2
|
|
image: busybox:1.36
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
if ! grep -q avx2 /proc/cpuinfo; then
|
|
echo "FATAL: this node's CPU has no avx2 flag." >&2
|
|
echo "registry.enkly.org/enkly/vllm-cpu (built with VLLM_CPU_DISABLE_AVX512=true) still needs AVX2 at minimum." >&2
|
|
echo "This is a much lower bar than the avx512f this cluster was already confirmed to lack, but if it's missing too, CPU inference isn't viable on this node." >&2
|
|
exit 1
|
|
fi
|
|
echo "avx2 present on this node — continuing (no avx512 requirement with this build)."
|
|
containers:
|
|
- name: vllm-chat
|
|
# Custom build, not public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo —
|
|
# see this file's header comment for why (confirmed missing
|
|
# avx512f) and the exact build/push command. This image must
|
|
# exist in your registry before applying.
|
|
image: registry.enkly.org/enkly/vllm-cpu:latest
|
|
args:
|
|
- --model=Qwen/Qwen2.5-1.5B-Instruct
|
|
- --served-model-name=$(CHAT_MODEL_NAME)
|
|
# Torch CPU float16 is unstable per vLLM's own CPU FAQ;
|
|
# bfloat16 is the documented safe default.
|
|
- --dtype=bfloat16
|
|
# Cut from 8192: the KV cache budget below is deliberately
|
|
# small to fit this node's real ~7.76Gi, and a shorter
|
|
# context is the lever that buys that back without
|
|
# dropping the KV cache to the point of thrashing.
|
|
- --max-model-len=4096
|
|
# Qwen2.5-Instruct's tool-call token format matches vLLM's
|
|
# hermes parser (the qwen3_coder parser the GPU deployment
|
|
# used is Qwen3.5-specific).
|
|
- --tool-call-parser=hermes
|
|
- --enable-auto-tool-choice
|
|
- --api-key=$(VLLM_API_KEY)
|
|
env:
|
|
- {name: CHAT_MODEL_NAME, valueFrom: {configMapKeyRef: {name: enkly-config, key: ENKLY_CHAT_MODEL}}}
|
|
- {name: VLLM_API_KEY, valueFrom: {secretKeyRef: {name: enkly-secrets, key: ENKLY_VLLM_API_KEY}}}
|
|
- {name: HUGGING_FACE_HUB_TOKEN, valueFrom: {secretKeyRef: {name: enkly-secrets, key: HUGGING_FACE_HUB_TOKEN}}}
|
|
# Cut from 4 — this plus ~3GB of resident bf16 weights is
|
|
# what resources.requests.memory below is actually sized
|
|
# against. Raise only after `kubectl describe node` shows
|
|
# real free RAM on top of what's already running, and
|
|
# raise the memory request/limit by the same amount.
|
|
- {name: VLLM_CPU_KVCACHE_SPACE, value: "1"}
|
|
# auto binds each OpenMP thread to a physical core and
|
|
# reserves VLLM_CPU_NUM_OF_RESERVED_CPU cores for the
|
|
# front-end process — vLLM's own recommended default for
|
|
# single-rank (no tensor/pipeline parallel) deployments
|
|
# like this one.
|
|
- {name: VLLM_CPU_OMP_THREADS_BIND, value: "auto"}
|
|
- {name: VLLM_CPU_NUM_OF_RESERVED_CPU, value: "1"}
|
|
ports: [{name: http, containerPort: 8000}]
|
|
resources:
|
|
# ~3GB resident for bf16 weights + 1GiB KV cache
|
|
# (VLLM_CPU_KVCACHE_SPACE above) + CPU-backend
|
|
# interpreter/runtime overhead (torch, OpenMP worker
|
|
# threads, inference graph buffers — typically 0.5-1GiB
|
|
# for a model this size) adds up to a realistic ~4.5Gi
|
|
# steady-state, not ~4Gi — the previous 3Gi *request*
|
|
# undersold that, which doesn't block scheduling (the
|
|
# scheduler only bin-packs on the request) but left too
|
|
# little daylight to the 5Gi *limit* during the kernel
|
|
# warmup/compile the startupProbe below already budgets
|
|
# extra time for, risking an OOMKill right at startup.
|
|
# 4Gi/6Gi still sits well under this cluster's ~7.76Gi
|
|
# per-node total (10-rabbitmq.yaml) with room for whatever
|
|
# else is already scheduled there — the original 8Gi
|
|
# request couldn't fit on *any* node here, full stop,
|
|
# which is why it sat Pending; this isn't a return to
|
|
# that. Raise only after `kubectl describe node` confirms
|
|
# real headroom.
|
|
requests: {cpu: "2", memory: 4Gi}
|
|
limits: {cpu: "3", memory: 6Gi}
|
|
startupProbe:
|
|
# No model-download wait this time on a warm PVC (unlike
|
|
# the brief Ollama detour's explicit pull step) — HF Hub
|
|
# download only happens once, cached on vllm-chat-cache
|
|
# after that. What's slow instead is the CPU backend's own
|
|
# kernel warmup/compile on every start, hence the still
|
|
# generous budget.
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 15
|
|
failureThreshold: 60 # 15 minutes
|
|
readinessProbe:
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 10
|
|
failureThreshold: 3
|
|
livenessProbe:
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 20
|
|
failureThreshold: 3
|
|
volumeMounts:
|
|
- {name: hf-cache, mountPath: /root/.cache/huggingface}
|
|
# vLLM's CPU docs still recommend a real /dev/shm for its
|
|
# multiprocessing workers, same reasoning as the GPU
|
|
# version had for NCCL — just a smaller budget since
|
|
# there's no multi-GPU scratch space involved here.
|
|
- {name: dshm, mountPath: /dev/shm}
|
|
volumes:
|
|
- name: hf-cache
|
|
persistentVolumeClaim: {claimName: vllm-chat-cache}
|
|
- name: dshm
|
|
emptyDir: {medium: Memory, sizeLimit: 1Gi}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: vllm-chat, namespace: enkly, labels: {app: vllm-chat}}
|
|
spec:
|
|
# ClusterIP only — no public Ingress this time. agent is the only
|
|
# caller and it's in-cluster; the original GPU manifest's
|
|
# vllm-chat.enkly.org Ingress existed for direct external testing,
|
|
# which isn't needed for this deployment to work. Add one back (same
|
|
# shape as 30-ingress.yaml) if external access is genuinely wanted.
|
|
selector: {app: vllm-chat}
|
|
ports: [{name: http, port: 8000, targetPort: http}]
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: vllm-embed, namespace: enkly, labels: {app: vllm-embed}}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: vllm-embed}}
|
|
template:
|
|
metadata: {labels: {app: vllm-embed}}
|
|
spec:
|
|
imagePullSecrets: [{name: enkly-registry}]
|
|
initContainers:
|
|
- name: check-cpu-avx2
|
|
image: busybox:1.36
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
if ! grep -q avx2 /proc/cpuinfo; then
|
|
echo "FATAL: this node's CPU has no avx2 flag." >&2
|
|
echo "registry.enkly.org/enkly/vllm-cpu (built with VLLM_CPU_DISABLE_AVX512=true) still needs AVX2 at minimum." >&2
|
|
exit 1
|
|
fi
|
|
echo "avx2 present on this node — continuing (no avx512 requirement with this build)."
|
|
containers:
|
|
- name: vllm-embed
|
|
image: registry.enkly.org/enkly/vllm-cpu:latest
|
|
args:
|
|
- --model=nomic-ai/nomic-embed-text-v1.5
|
|
- --served-model-name=$(EMBED_MODEL_NAME)
|
|
- --task=embed
|
|
# Nomic's embedding model config uses custom (non-standard
|
|
# transformers) modeling code — needs explicit opt-in.
|
|
- --trust-remote-code
|
|
- --dtype=bfloat16
|
|
- --api-key=$(VLLM_API_KEY)
|
|
env:
|
|
- {name: EMBED_MODEL_NAME, valueFrom: {configMapKeyRef: {name: enkly-config, key: ENKLY_EMBED_MODEL}}}
|
|
- {name: VLLM_API_KEY, valueFrom: {secretKeyRef: {name: enkly-secrets, key: ENKLY_VLLM_API_KEY}}}
|
|
- {name: HUGGING_FACE_HUB_TOKEN, valueFrom: {secretKeyRef: {name: enkly-secrets, key: HUGGING_FACE_HUB_TOKEN}}}
|
|
# A 137M-parameter embedding model needs far less KV cache
|
|
# headroom than the chat model.
|
|
- {name: VLLM_CPU_KVCACHE_SPACE, value: "1"}
|
|
- {name: VLLM_CPU_OMP_THREADS_BIND, value: "auto"}
|
|
- {name: VLLM_CPU_NUM_OF_RESERVED_CPU, value: "1"}
|
|
ports: [{name: http, containerPort: 8000}]
|
|
resources:
|
|
# Trimmed from 2Gi/4Gi for the same per-node headroom
|
|
# reason as vllm-chat above — nomic-embed-text-v1.5 is
|
|
# ~550MB on disk, so even this is generous.
|
|
requests: {cpu: "1", memory: 1Gi}
|
|
limits: {cpu: "2", memory: 2Gi}
|
|
startupProbe:
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 10
|
|
failureThreshold: 36 # 6 minutes — small model, no large download after first boot
|
|
readinessProbe:
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 10
|
|
failureThreshold: 3
|
|
livenessProbe:
|
|
httpGet: {path: /health, port: http}
|
|
periodSeconds: 20
|
|
failureThreshold: 3
|
|
volumeMounts:
|
|
- {name: hf-cache, mountPath: /root/.cache/huggingface}
|
|
- {name: dshm, mountPath: /dev/shm}
|
|
volumes:
|
|
- name: hf-cache
|
|
persistentVolumeClaim: {claimName: vllm-embed-cache}
|
|
- name: dshm
|
|
emptyDir: {medium: Memory, sizeLimit: 256Mi}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: vllm-embed, namespace: enkly, labels: {app: vllm-embed}}
|
|
spec:
|
|
selector: {app: vllm-embed}
|
|
ports: [{name: http, port: 8000, targetPort: http}]
|