name: marlin-4090-gemma4-e4b

# Official vLLM Gemma 4 image on Docker Hub. No Ray, no Vast portal stack. Paste
# parts of this into Vast.AI UI or equivalent, or automate this using their API

registry: docker
image: vllm/vllm-openai:gemma4
tag: they offset tags pinned at OS and CUDA versions

# Jupyter or SSH mode means vastai overwrites the containers entrypoint.
# With SSH mode, we must call whatever the images entrypoint used to call
mode: ssh

# ~10GB compressed image + Gemma 4 E4B weights + HF/cache headroom
disk_gb: 40

# API listens on 127.0.0.1:8000 inside the instance.
# From your PC: ssh -i ~/.ssh/<private_key_file> -p <port> root@<ip> -L 8000:127.0.0.1:8000
ports: 8000

# HF_TOKEN is recommended for faster downloads and access to some restricted models.
# Gemma 4 is not restricted
env:
  HF_TOKEN=<hugging-face-token>
  HF_HOME=/workspace/hf

startup: |
  #!/usr/bin/env bash
  set -euo pipefail

  mkdir -p /workspace/models /workspace/hf
  export HF_TOKEN=
  export HF_HOME="${HF_HOME:-/workspace/hf}"
  export HUGGING_FACE_HUB_TOKEN="${HUGGING_FACE_HUB_TOKEN:-${HF_TOKEN:-}}"

  # fix bug "transformers.integrations.heterogeneity.configuration_utils.AmbiguousGlobalPerLayerAttributeError: 'head_dim' is a per-layer attribute and may vary across layers"
  pip install 'transformers==5.14.1' --break-system-packages

  # On Vast on-start, invoke explicitly:
  exec vllm serve google/gemma-4-E4B-it \
    --host 127.0.0.1 \
    --port 8000 \
    --max-model-len 5184 \
    --max-num-seqs 32 \
    --max-num-batched-tokens 8192 \
    --gpu-memory-utilization 0.88 \
    --kv-cache-dtype fp8 \
    --enable-prefix-caching \
    --async-scheduling \
    --uvicorn-log-level warning \
    --download-dir /workspace/models

# For Vast.AI specifically https://docs.vast.ai/cli/reference/search-offers#available-fields
filters: num_gpus=1 gpu_ram>=24000

notes: |
  Prefer dedicated/faster CPU over inflated "32 vCPU" shares
  Ensure GPU CUDA version is compatible with the image tag you choose

  Smoke on the box:
    curl -s --max-time 5 http://127.0.0.1:8000/v1/models

  Marlin worker profile (vast-g4-4b-1) after you've ran the SSH tunnel

  Flag rationale (ops, not quality):
    max-model-len 5184  — keep margin; catalog policy prompts p99 ~1900 + max_tokens 400 (v1-simple)
    max-num-seqs 32     — match worker concurrency, increase if the box can handle it
    batched-tokens 8192 — cap prefill-wall / activation spikes
    util 0.88           — leave activation headroom (KV is usually <<50%)
    no Ray              — avoids template CPU busy-wait on 1 GPU