211 lines
9.4 KiB
YAML
211 lines
9.4 KiB
YAML
name: Go CI Pipeline
|
|
on: [push, pull_request]
|
|
|
|
jobs:
|
|
build_and_test:
|
|
runs-on: ubuntu-latest
|
|
|
|
# Start a Docker-in-Docker service container
|
|
services:
|
|
docker:
|
|
image: docker:dind
|
|
# --dns pins the dind container to reliable public resolvers instead
|
|
# of whatever it inherits from the runner host. Without this, the
|
|
# embedded 127.0.0.11 resolver inside dind intermittently fails to
|
|
# forward lookups for the registry host ("server misbehaving"),
|
|
# which otherwise kills `docker push` nondeterministically.
|
|
options: --privileged --dns 1.1.1.1 --dns 8.8.8.8
|
|
env:
|
|
DOCKER_TLS_CERTDIR: ""
|
|
|
|
# Point docker CLI to the dind container
|
|
env:
|
|
DOCKER_HOST: tcp://docker:2375
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v5
|
|
with:
|
|
go-version: "1.23"
|
|
|
|
- name: Install dependencies
|
|
run: go mod tidy && go mod download
|
|
|
|
- name: Vet and test
|
|
run: |
|
|
go vet ./...
|
|
go test ./...
|
|
|
|
- name: Login to private registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ secrets.REGISTRY_URL }}
|
|
username: ${{ secrets.REGISTRY_USERNAME }}
|
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
|
|
|
- name: Build and push container images
|
|
run: |
|
|
set -e
|
|
REG="${{ secrets.REGISTRY_URL }}/enkly"
|
|
SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
|
|
|
|
# Retry helper: docker push against the dind service occasionally
|
|
# hits a transient "server misbehaving" DNS failure (127.0.0.11
|
|
# couldn't resolve the registry host) rather than a real error.
|
|
# Retry a few times with backoff before giving up for real.
|
|
push_with_retry() {
|
|
local ref="$1"
|
|
local attempt=1
|
|
local max_attempts=5
|
|
until docker push "$ref"; do
|
|
if [ "$attempt" -ge "$max_attempts" ]; then
|
|
echo "::error::docker push $ref failed after $max_attempts attempts"
|
|
return 1
|
|
fi
|
|
local wait=$((attempt * 5))
|
|
echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
|
|
sleep "$wait"
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
|
|
for svc in iam billing conversation workspace knowledge agent notification gateway; do
|
|
docker build -f services/$svc/Dockerfile -t "$REG/$svc:$SHA" -t "$REG/$svc:latest" .
|
|
push_with_retry "$REG/$svc:$SHA"
|
|
push_with_retry "$REG/$svc:latest"
|
|
done
|
|
|
|
- name: Build and push vLLM CPU (no AVX512) image
|
|
# This is NOT one of the 8 enkly service images above — it's the
|
|
# custom vllm/vllm-openai build that k8s/11-vllm.yaml's vllm-chat
|
|
# and vllm-embed Deployments require instead of the official
|
|
# public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo image, because
|
|
# that pre-built image hard-requires avx512f (missing on the
|
|
# target cluster's nodes) and crashes with "Illegal instruction"
|
|
# without it. Source and build args match k8s/11-vllm.yaml's and
|
|
# k8s/README.md's documented command, plus an explicit max_jobs
|
|
# cap explained below.
|
|
#
|
|
# Root cause of the previous failure ("Cannot connect to the
|
|
# Docker daemon at tcp://localhost:2375" mid-build): vLLM's
|
|
# docker/Dockerfile.cpu defaults to `ARG max_jobs=32`, i.e. 32
|
|
# parallel C++ translation units (oneDNN/cutlass kernels etc.)
|
|
# compiling at once, each easily using >1GB RES. The dind
|
|
# service here — like the target cluster's own nodes (see
|
|
# k8s/README.md: ~7949Mi/~7.76Gi per node) — only has a few GB
|
|
# of RAM and 2 vCPUs, nowhere near enough for 32-way parallel
|
|
# compilation. The build log showed exactly this: individual
|
|
# objects going from ~1-10s to 317s (heavy swapping) right
|
|
# before the dind container's dockerd was OOM-killed, which
|
|
# drops the daemon out from under the still-running `docker
|
|
# build` and surfaces as a generic "cannot connect" error rather
|
|
# than an explicit OOM message. A plain retry doesn't fix this —
|
|
# dockerd itself died, so the service container is gone — only
|
|
# bounding parallelism does. We size MAX_JOBS off the runner's
|
|
# actual CPU count and memory (~2GB headroom per job) instead of
|
|
# trusting the Dockerfile's hardcoded default, floor 1.
|
|
#
|
|
# Caveat: this clones and compiles vLLM from source on every run
|
|
# of this job, on a small runner — expect this step alone to
|
|
# take a long time and to be the slowest part of the pipeline.
|
|
# If that becomes a problem, the usual fixes are: only run this
|
|
# step on a path filter scoped to a pinned vLLM version/tag
|
|
# instead of vLLM's default branch, add Docker layer caching
|
|
# (e.g. actions/cache or a registry-backed BuildKit cache), or
|
|
# move it to its own workflow that isn't triggered on every
|
|
# push/PR. We already pin the clone to a tagged vLLM release
|
|
# below so a build is reproducible and isn't broken by upstream
|
|
# `main` changing under us.
|
|
run: |
|
|
set -e
|
|
REG="${{ secrets.REGISTRY_URL }}/enkly"
|
|
SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
|
|
VLLM_REF="v0.8.5"
|
|
|
|
push_with_retry() {
|
|
local ref="$1"
|
|
local attempt=1
|
|
local max_attempts=5
|
|
until docker push "$ref"; do
|
|
if [ "$attempt" -ge "$max_attempts" ]; then
|
|
echo "::error::docker push $ref failed after $max_attempts attempts"
|
|
return 1
|
|
fi
|
|
local wait=$((attempt * 5))
|
|
echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
|
|
sleep "$wait"
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
|
|
# Make sure the dind daemon is actually reachable before we
|
|
# kick off a build that can run for tens of minutes.
|
|
for i in $(seq 1 30); do
|
|
docker info >/dev/null 2>&1 && break
|
|
echo "waiting for docker daemon ($i/30)..."
|
|
sleep 2
|
|
done
|
|
docker info >/dev/null 2>&1 || { echo "::error::docker daemon at $DOCKER_HOST is not reachable"; exit 1; }
|
|
|
|
# Cap compile parallelism to what this runner can actually
|
|
# hold in memory, instead of the Dockerfile's hardcoded
|
|
# max_jobs=32 (see note above for why that OOM-kills dockerd).
|
|
# Budget ~2GB per compile job and never exceed the vCPU count.
|
|
CPU_COUNT="$(nproc 2>/dev/null || echo 2)"
|
|
MEM_KB="$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)"
|
|
MEM_JOBS=$((MEM_KB / 1024 / 1024 / 2))
|
|
MAX_JOBS="$CPU_COUNT"
|
|
if [ "$MEM_JOBS" -gt 0 ] && [ "$MEM_JOBS" -lt "$MAX_JOBS" ]; then
|
|
MAX_JOBS="$MEM_JOBS"
|
|
fi
|
|
[ "$MAX_JOBS" -lt 1 ] && MAX_JOBS=1
|
|
echo "Building vLLM with MAX_JOBS=$MAX_JOBS (cpu=$CPU_COUNT, mem_kb=$MEM_KB)"
|
|
|
|
# `git clone` below runs directly on the runner host (unlike
|
|
# `docker build`/push, it never touches $DOCKER_HOST), so the
|
|
# --dns 1.1.1.1/8.8.8.8 pin on the dind *service* above does
|
|
# nothing for it — that only fixes resolution for things the
|
|
# dind daemon itself does (image pulls, registry push). A
|
|
# host-level resolver hiccup ("Could not resolve host:
|
|
# github.com") is the same class of transient DNS flakiness
|
|
# called out in the dind `options` comment, just hitting a
|
|
# different resolver. Retry with backoff, and nudge the host
|
|
# resolver between attempts where we can (best-effort, never
|
|
# fatal if the tool/permission isn't there).
|
|
clone_with_retry() {
|
|
local attempt=1
|
|
local max_attempts=5
|
|
until git clone --depth 1 --branch "$VLLM_REF" https://github.com/vllm-project/vllm.git /tmp/vllm-build; do
|
|
if [ "$attempt" -ge "$max_attempts" ]; then
|
|
echo "::error::git clone of vllm-project/vllm failed after $max_attempts attempts"
|
|
return 1
|
|
fi
|
|
local wait=$((attempt * 5))
|
|
echo "git clone failed (attempt $attempt/$max_attempts, likely DNS); retrying in ${wait}s..."
|
|
sleep "$wait"
|
|
# Best-effort: if the host resolver cache is stuck, poke it.
|
|
# Ignore failures - these commands may not exist/be permitted.
|
|
(command -v resolvectl >/dev/null 2>&1 && sudo resolvectl flush-caches) || true
|
|
(command -v systemd-resolve >/dev/null 2>&1 && sudo systemd-resolve --flush-caches) || true
|
|
getent hosts github.com >/dev/null 2>&1 || true
|
|
rm -rf /tmp/vllm-build
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
|
|
rm -rf /tmp/vllm-build
|
|
clone_with_retry
|
|
|
|
docker build -f /tmp/vllm-build/docker/Dockerfile.cpu \
|
|
--build-arg VLLM_CPU_DISABLE_AVX512=true \
|
|
--build-arg max_jobs="$MAX_JOBS" \
|
|
--tag "$REG/vllm-cpu:$SHA" \
|
|
--tag "$REG/vllm-cpu:latest" \
|
|
--target vllm-openai \
|
|
/tmp/vllm-build
|
|
|
|
push_with_retry "$REG/vllm-cpu:$SHA"
|
|
push_with_retry "$REG/vllm-cpu:latest" |