name: Go CI Pipeline
on: [push, pull_request]

jobs:
  build_and_test:
    runs-on: ubuntu-latest

    # Start a Docker-in-Docker service container
    services:
      docker:
        image: docker:dind
        # --dns pins the dind container to reliable public resolvers instead
        # of whatever it inherits from the runner host. Without this, the
        # embedded 127.0.0.11 resolver inside dind intermittently fails to
        # forward lookups for the registry host ("server misbehaving"),
        # which otherwise kills `docker push` nondeterministically.
        options: --privileged --dns 1.1.1.1 --dns 8.8.8.8
        env:
          DOCKER_TLS_CERTDIR: ""

    # Point docker CLI to the dind container
    env:
      DOCKER_HOST: tcp://docker:2375

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.23"

      - name: Install dependencies
        run: go mod tidy && go mod download

      - name: Vet and test
        run: |
          go vet ./...
          go test ./...

      - name: Login to private registry
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.REGISTRY_URL }}
          username: ${{ secrets.REGISTRY_USERNAME }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: Build and push container images
        run: |
          set -e
          REG="${{ secrets.REGISTRY_URL }}/enkly"
          SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"

          # Retry helper: docker push against the dind service occasionally
          # hits a transient "server misbehaving" DNS failure (127.0.0.11
          # couldn't resolve the registry host) rather than a real error.
          # Retry a few times with backoff before giving up for real.
          push_with_retry() {
            local ref="$1"
            local attempt=1
            local max_attempts=5
            until docker push "$ref"; do
              if [ "$attempt" -ge "$max_attempts" ]; then
                echo "::error::docker push $ref failed after $max_attempts attempts"
                return 1
              fi
              local wait=$((attempt * 5))
              echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
              sleep "$wait"
              attempt=$((attempt + 1))
            done
          }

          for svc in iam billing conversation workspace knowledge agent notification gateway; do
            docker build -f services/$svc/Dockerfile -t "$REG/$svc:$SHA" -t "$REG/$svc:latest" .
            push_with_retry "$REG/$svc:$SHA"
            push_with_retry "$REG/$svc:latest"
          done

      - name: Build and push vLLM CPU (no AVX512) image
        # This is NOT one of the 8 enkly service images above — it's the
        # custom vllm/vllm-openai build that k8s/11-vllm.yaml's vllm-chat
        # and vllm-embed Deployments require instead of the official
        # public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo image, because
        # that pre-built image hard-requires avx512f (missing on the
        # target cluster's nodes) and crashes with "Illegal instruction"
        # without it. Source and build args match k8s/11-vllm.yaml's and
        # k8s/README.md's documented command, plus an explicit max_jobs
        # cap explained below.
        #
        # Root cause of the previous failure ("Cannot connect to the
        # Docker daemon at tcp://localhost:2375" mid-build): vLLM's
        # docker/Dockerfile.cpu defaults to `ARG max_jobs=32`, i.e. 32
        # parallel C++ translation units (oneDNN/cutlass kernels etc.)
        # compiling at once, each easily using >1GB RES. The dind
        # service here — like the target cluster's own nodes (see
        # k8s/README.md: ~7949Mi/~7.76Gi per node) — only has a few GB
        # of RAM and 2 vCPUs, nowhere near enough for 32-way parallel
        # compilation. The build log showed exactly this: individual
        # objects going from ~1-10s to 317s (heavy swapping) right
        # before the dind container's dockerd was OOM-killed, which
        # drops the daemon out from under the still-running `docker
        # build` and surfaces as a generic "cannot connect" error rather
        # than an explicit OOM message. A plain retry doesn't fix this —
        # dockerd itself died, so the service container is gone — only
        # bounding parallelism does. We size MAX_JOBS off the runner's
        # actual CPU count and memory (~2GB headroom per job) instead of
        # trusting the Dockerfile's hardcoded default, floor 1.
        #
        # Caveat: this clones and compiles vLLM from source on every run
        # of this job, on a small runner — expect this step alone to
        # take a long time and to be the slowest part of the pipeline.
        # If that becomes a problem, the usual fixes are: only run this
        # step on a path filter scoped to a pinned vLLM version/tag
        # instead of vLLM's default branch, add Docker layer caching
        # (e.g. actions/cache or a registry-backed BuildKit cache), or
        # move it to its own workflow that isn't triggered on every
        # push/PR. We already pin the clone to a tagged vLLM release
        # below so a build is reproducible and isn't broken by upstream
        # `main` changing under us.
        run: |
          set -e
          REG="${{ secrets.REGISTRY_URL }}/enkly"
          SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
          VLLM_REF="v0.8.5"

          push_with_retry() {
            local ref="$1"
            local attempt=1
            local max_attempts=5
            until docker push "$ref"; do
              if [ "$attempt" -ge "$max_attempts" ]; then
                echo "::error::docker push $ref failed after $max_attempts attempts"
                return 1
              fi
              local wait=$((attempt * 5))
              echo "push of $ref failed (attempt $attempt/$max_attempts); retrying in ${wait}s..."
              sleep "$wait"
              attempt=$((attempt + 1))
            done
          }

          # Make sure the dind daemon is actually reachable before we
          # kick off a build that can run for tens of minutes.
          for i in $(seq 1 30); do
            docker info >/dev/null 2>&1 && break
            echo "waiting for docker daemon ($i/30)..."
            sleep 2
          done
          docker info >/dev/null 2>&1 || { echo "::error::docker daemon at $DOCKER_HOST is not reachable"; exit 1; }

          # Cap compile parallelism to what this runner can actually
          # hold in memory, instead of the Dockerfile's hardcoded
          # max_jobs=32 (see note above for why that OOM-kills dockerd).
          # Budget ~2GB per compile job and never exceed the vCPU count.
          CPU_COUNT="$(nproc 2>/dev/null || echo 2)"
          MEM_KB="$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)"
          MEM_JOBS=$((MEM_KB / 1024 / 1024 / 2))
          MAX_JOBS="$CPU_COUNT"
          if [ "$MEM_JOBS" -gt 0 ] && [ "$MEM_JOBS" -lt "$MAX_JOBS" ]; then
            MAX_JOBS="$MEM_JOBS"
          fi
          [ "$MAX_JOBS" -lt 1 ] && MAX_JOBS=1
          echo "Building vLLM with MAX_JOBS=$MAX_JOBS (cpu=$CPU_COUNT, mem_kb=$MEM_KB)"

          # `git clone` below runs directly on the runner host (unlike
          # `docker build`/push, it never touches $DOCKER_HOST), so the
          # --dns 1.1.1.1/8.8.8.8 pin on the dind *service* above does
          # nothing for it — that only fixes resolution for things the
          # dind daemon itself does (image pulls, registry push). A
          # host-level resolver hiccup ("Could not resolve host:
          # github.com") is the same class of transient DNS flakiness
          # called out in the dind `options` comment, just hitting a
          # different resolver. Retry with backoff, and nudge the host
          # resolver between attempts where we can (best-effort, never
          # fatal if the tool/permission isn't there).
          clone_with_retry() {
            local attempt=1
            local max_attempts=5
            until git clone --depth 1 --branch "$VLLM_REF" https://github.com/vllm-project/vllm.git /tmp/vllm-build; do
              if [ "$attempt" -ge "$max_attempts" ]; then
                echo "::error::git clone of vllm-project/vllm failed after $max_attempts attempts"
                return 1
              fi
              local wait=$((attempt * 5))
              echo "git clone failed (attempt $attempt/$max_attempts, likely DNS); retrying in ${wait}s..."
              sleep "$wait"
              # Best-effort: if the host resolver cache is stuck, poke it.
              # Ignore failures - these commands may not exist/be permitted.
              (command -v resolvectl >/dev/null 2>&1 && sudo resolvectl flush-caches) || true
              (command -v systemd-resolve >/dev/null 2>&1 && sudo systemd-resolve --flush-caches) || true
              getent hosts github.com >/dev/null 2>&1 || true
              rm -rf /tmp/vllm-build
              attempt=$((attempt + 1))
            done
          }

          rm -rf /tmp/vllm-build
          clone_with_retry

          docker build -f /tmp/vllm-build/docker/Dockerfile.cpu \
            --build-arg VLLM_CPU_DISABLE_AVX512=true \
            --build-arg max_jobs="$MAX_JOBS" \
            --tag "$REG/vllm-cpu:$SHA" \
            --tag "$REG/vllm-cpu:latest" \
            --target vllm-openai \
            /tmp/vllm-build

          push_with_retry "$REG/vllm-cpu:$SHA"
          push_with_retry "$REG/vllm-cpu:latest"