This commit is contained in:
enkly
211
.gitea/workflows/ci.yaml
Normal file
211
.gitea/workflows/ci.yaml
Normal file
@@ -0,0 +1,211 @@
|
||||
name: Go CI Pipeline
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build_and_test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Start a Docker-in-Docker service container
|
||||
services:
|
||||
docker:
|
||||
image: docker:dind
|
||||
# --dns pins the dind container to reliable public resolvers instead
|
||||
# of whatever it inherits from the runner host. Without this, the
|
||||
# embedded 127.0.0.11 resolver inside dind intermittently fails to
|
||||
# forward lookups for the registry host ("server misbehaving"),
|
||||
# which otherwise kills `docker push` nondeterministically.
|
||||
options: --privileged --dns 1.1.1.1 --dns 8.8.8.8
|
||||
env:
|
||||
DOCKER_TLS_CERTDIR: ""
|
||||
|
||||
# Point docker CLI to the dind container
|
||||
env:
|
||||
DOCKER_HOST: tcp://docker:2375
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install dependencies
|
||||
run: go mod tidy && go mod download
|
||||
|
||||
- name: Vet and test
|
||||
run: |
|
||||
go vet ./...
|
||||
go test ./...
|
||||
|
||||
- name: Login to private registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ secrets.REGISTRY_URL }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- name: Build and push container images
|
||||
run: |
|
||||
set -e
|
||||
REG="${{ secrets.REGISTRY_URL }}/enkly"
|
||||
SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
|
||||
|
||||
# Retry helper: docker push against the dind service occasionally
|
||||
# hits a transient "server misbehaving" DNS failure (127.0.0.11
|
||||
# couldn't resolve the registry host) rather than a real error.
|
||||
# Retry a few times with backoff before giving up for real.
|
||||
push_with_retry() {
|
||||
local ref="$1"
|
||||
local attempt=1
|
||||
local max_attempts=5
|
||||
until docker push "$ref"; do
|
||||
if [ "$attempt" -ge "$max_attempts" ]; then
|
||||
echo "::error::docker push $ref failed after $max_attempts attempts"
|
||||
return 1
|
||||
fi
|
||||
local wait=$((attempt * 5))
|
||||
echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
|
||||
sleep "$wait"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
}
|
||||
|
||||
for svc in iam billing conversation workspace knowledge agent notification gateway; do
|
||||
docker build -f services/$svc/Dockerfile -t "$REG/$svc:$SHA" -t "$REG/$svc:latest" .
|
||||
push_with_retry "$REG/$svc:$SHA"
|
||||
push_with_retry "$REG/$svc:latest"
|
||||
done
|
||||
|
||||
- name: Build and push vLLM CPU (no AVX512) image
|
||||
# This is NOT one of the 8 enkly service images above — it's the
|
||||
# custom vllm/vllm-openai build that k8s/11-vllm.yaml's vllm-chat
|
||||
# and vllm-embed Deployments require instead of the official
|
||||
# public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo image, because
|
||||
# that pre-built image hard-requires avx512f (missing on the
|
||||
# target cluster's nodes) and crashes with "Illegal instruction"
|
||||
# without it. Source and build args match k8s/11-vllm.yaml's and
|
||||
# k8s/README.md's documented command, plus an explicit max_jobs
|
||||
# cap explained below.
|
||||
#
|
||||
# Root cause of the previous failure ("Cannot connect to the
|
||||
# Docker daemon at tcp://localhost:2375" mid-build): vLLM's
|
||||
# docker/Dockerfile.cpu defaults to `ARG max_jobs=32`, i.e. 32
|
||||
# parallel C++ translation units (oneDNN/cutlass kernels etc.)
|
||||
# compiling at once, each easily using >1GB RES. The dind
|
||||
# service here — like the target cluster's own nodes (see
|
||||
# k8s/README.md: ~7949Mi/~7.76Gi per node) — only has a few GB
|
||||
# of RAM and 2 vCPUs, nowhere near enough for 32-way parallel
|
||||
# compilation. The build log showed exactly this: individual
|
||||
# objects going from ~1-10s to 317s (heavy swapping) right
|
||||
# before the dind container's dockerd was OOM-killed, which
|
||||
# drops the daemon out from under the still-running `docker
|
||||
# build` and surfaces as a generic "cannot connect" error rather
|
||||
# than an explicit OOM message. A plain retry doesn't fix this —
|
||||
# dockerd itself died, so the service container is gone — only
|
||||
# bounding parallelism does. We size MAX_JOBS off the runner's
|
||||
# actual CPU count and memory (~2GB headroom per job) instead of
|
||||
# trusting the Dockerfile's hardcoded default, floor 1.
|
||||
#
|
||||
# Caveat: this clones and compiles vLLM from source on every run
|
||||
# of this job, on a small runner — expect this step alone to
|
||||
# take a long time and to be the slowest part of the pipeline.
|
||||
# If that becomes a problem, the usual fixes are: only run this
|
||||
# step on a path filter scoped to a pinned vLLM version/tag
|
||||
# instead of vLLM's default branch, add Docker layer caching
|
||||
# (e.g. actions/cache or a registry-backed BuildKit cache), or
|
||||
# move it to its own workflow that isn't triggered on every
|
||||
# push/PR. We already pin the clone to a tagged vLLM release
|
||||
# below so a build is reproducible and isn't broken by upstream
|
||||
# `main` changing under us.
|
||||
run: |
|
||||
set -e
|
||||
REG="${{ secrets.REGISTRY_URL }}/enkly"
|
||||
SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
|
||||
VLLM_REF="v0.8.5"
|
||||
|
||||
push_with_retry() {
|
||||
local ref="$1"
|
||||
local attempt=1
|
||||
local max_attempts=5
|
||||
until docker push "$ref"; do
|
||||
if [ "$attempt" -ge "$max_attempts" ]; then
|
||||
echo "::error::docker push $ref failed after $max_attempts attempts"
|
||||
return 1
|
||||
fi
|
||||
local wait=$((attempt * 5))
|
||||
echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
|
||||
sleep "$wait"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# Make sure the dind daemon is actually reachable before we
|
||||
# kick off a build that can run for tens of minutes.
|
||||
for i in $(seq 1 30); do
|
||||
docker info >/dev/null 2>&1 && break
|
||||
echo "waiting for docker daemon ($i/30)..."
|
||||
sleep 2
|
||||
done
|
||||
docker info >/dev/null 2>&1 || { echo "::error::docker daemon at $DOCKER_HOST is not reachable"; exit 1; }
|
||||
|
||||
# Cap compile parallelism to what this runner can actually
|
||||
# hold in memory, instead of the Dockerfile's hardcoded
|
||||
# max_jobs=32 (see note above for why that OOM-kills dockerd).
|
||||
# Budget ~2GB per compile job and never exceed the vCPU count.
|
||||
CPU_COUNT="$(nproc 2>/dev/null || echo 2)"
|
||||
MEM_KB="$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)"
|
||||
MEM_JOBS=$((MEM_KB / 1024 / 1024 / 2))
|
||||
MAX_JOBS="$CPU_COUNT"
|
||||
if [ "$MEM_JOBS" -gt 0 ] && [ "$MEM_JOBS" -lt "$MAX_JOBS" ]; then
|
||||
MAX_JOBS="$MEM_JOBS"
|
||||
fi
|
||||
[ "$MAX_JOBS" -lt 1 ] && MAX_JOBS=1
|
||||
echo "Building vLLM with MAX_JOBS=$MAX_JOBS (cpu=$CPU_COUNT, mem_kb=$MEM_KB)"
|
||||
|
||||
# `git clone` below runs directly on the runner host (unlike
|
||||
# `docker build`/push, it never touches $DOCKER_HOST), so the
|
||||
# --dns 1.1.1.1/8.8.8.8 pin on the dind *service* above does
|
||||
# nothing for it — that only fixes resolution for things the
|
||||
# dind daemon itself does (image pulls, registry push). A
|
||||
# host-level resolver hiccup ("Could not resolve host:
|
||||
# github.com") is the same class of transient DNS flakiness
|
||||
# called out in the dind `options` comment, just hitting a
|
||||
# different resolver. Retry with backoff, and nudge the host
|
||||
# resolver between attempts where we can (best-effort, never
|
||||
# fatal if the tool/permission isn't there).
|
||||
clone_with_retry() {
|
||||
local attempt=1
|
||||
local max_attempts=5
|
||||
until git clone --depth 1 --branch "$VLLM_REF" https://github.com/vllm-project/vllm.git /tmp/vllm-build; do
|
||||
if [ "$attempt" -ge "$max_attempts" ]; then
|
||||
echo "::error::git clone of vllm-project/vllm failed after $max_attempts attempts"
|
||||
return 1
|
||||
fi
|
||||
local wait=$((attempt * 5))
|
||||
echo "git clone failed (attempt $attempt/$max_attempts, likely DNS); retrying in ${wait}s..."
|
||||
sleep "$wait"
|
||||
# Best-effort: if the host resolver cache is stuck, poke it.
|
||||
# Ignore failures - these commands may not exist/be permitted.
|
||||
(command -v resolvectl >/dev/null 2>&1 && sudo resolvectl flush-caches) || true
|
||||
(command -v systemd-resolve >/dev/null 2>&1 && sudo systemd-resolve --flush-caches) || true
|
||||
getent hosts github.com >/dev/null 2>&1 || true
|
||||
rm -rf /tmp/vllm-build
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
}
|
||||
|
||||
rm -rf /tmp/vllm-build
|
||||
clone_with_retry
|
||||
|
||||
docker build -f /tmp/vllm-build/docker/Dockerfile.cpu \
|
||||
--build-arg VLLM_CPU_DISABLE_AVX512=true \
|
||||
--build-arg max_jobs="$MAX_JOBS" \
|
||||
--tag "$REG/vllm-cpu:$SHA" \
|
||||
--tag "$REG/vllm-cpu:latest" \
|
||||
--target vllm-openai \
|
||||
/tmp/vllm-build
|
||||
|
||||
push_with_retry "$REG/vllm-cpu:$SHA"
|
||||
push_with_retry "$REG/vllm-cpu:latest"
|
||||
Reference in New Issue
Block a user