# LiteLLM proxy config: local first, cloud only by explicit rule.
#
# Kit template for guide doc 1 (layer L3) and doc 2 (build step "LiteLLM"). Checked against the LiteLLM
# docs as of October 2026 (v1.105.1): proxy/configs, routing, providers/ollama, proxy/caching.
# Mount it read-only at /app/config.yaml (see ../core/docker-compose.yml). Secrets come from the
# environment (os.environ/NAME); never write a key into this file.
#
# Aliases callers use:
#   triage   tiny local model for routing and short JSON (about 0.4 s on a 24-core CPU)
#   extract  8B local model for JSON extraction and short drafts (about 3 s median on short cases)
#   embed    local embedding model, 768 dimensions
#   cloud-long  a hosted model for inputs too long for the local context. Only keys that list it
#               may use it; give those keys a budget. Delete this entry if nothing may leave the server.

model_list:
  - model_name: triage
    litellm_params:
      model: ollama_chat/qwen2.5:1.5b
      api_base: http://ollama:11434
      keep_alive: "10m"
      timeout: 60
    model_info:
      input_cost_per_token: 0
      output_cost_per_token: 0

  - model_name: extract
    litellm_params:
      model: ollama_chat/hermes3:8b
      api_base: http://ollama:11434
      keep_alive: "15m"
      timeout: 180
    model_info:
      input_cost_per_token: 0
      output_cost_per_token: 0

  - model_name: embed
    litellm_params:
      model: ollama/nomic-embed-text
      api_base: http://ollama:11434
    model_info:
      mode: embedding
      input_cost_per_token: 0

  - model_name: cloud-long
    litellm_params:
      model: anthropic/<model-id>              # or openai/<model-id>; check the provider's current list
      api_key: os.environ/ANTHROPIC_API_KEY
      timeout: 300

router_settings:
  routing_strategy: simple-shuffle           # the documented default
  num_retries: 1
  timeout: 300
  allowed_fails: 3
  cooldown_time: 30
  fallbacks:
    - triage: ["extract"]                    # tiny model failed or timed out: try the 8B, still local
  context_window_fallbacks:
    - extract: ["cloud-long"]                # only when the input does not fit locally

litellm_settings:
  drop_params: true                          # drop parameters a backend does not support
  cache: true
  cache_params:
    type: local                              # in-memory exact-match cache, one gateway instance
    # Semantic cache in Qdrant instead (warning from the docs: semantic caches can replay stale
    # answers on multi-turn or agent traffic, so use them for one-shot lookups only):
    # type: qdrant-semantic
    # qdrant_semantic_cache_embedding_model: embed
    # qdrant_collection_name: litellm_semantic_cache
    # qdrant_quantization_config: binary
    # qdrant_semantic_cache_vector_size: 768
    # similarity_threshold: 0.8

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY  # admin only; callers get virtual keys
  database_url: os.environ/DATABASE_URL      # keys, budgets and spend logs live here
  store_prompts_in_spend_logs: false         # count tokens and cost, never keep prompt text

# Give every caller its own virtual key, limited to the models it needs, with a budget.
# Example (run on the server; the master key is read from a file, never typed into a chat):
#
#   curl -s http://127.0.0.1:4000/key/generate \
#     -H "Authorization: Bearer $(cat ~/.config/litellm/master_key)" \
#     -H "Content-Type: application/json" \
#     -d '{"models": ["triage", "extract", "embed"], "key_alias": "prompt-hub"}'
#   (local models cost nothing per call, so the model list is the limit for this key)
#
#   curl -s http://127.0.0.1:4000/key/generate \
#     -H "Authorization: Bearer $(cat ~/.config/litellm/master_key)" \
#     -H "Content-Type: application/json" \
#     -d '{"models": ["extract", "cloud-long"], "max_budget": 20, "budget_duration": "30d", "key_alias": "nightly-analysis"}'