LiteLLM

LLM Gateway to call 100+ LLM APIs in OpenAI format

Python SDK, Proxy Server (LLM Gateway) to call 100+ LLM APIs in OpenAI format - [Bedrock, Azure, OpenAI, VertexAI, Cohere, Anthropic, Sagemaker, HuggingFace, Replicate, Groq]

Links:

Home: https://docs.litellm.ai/
Source: https://github.com/BerriAI/litellm
Proxy Deploy: https://docs.litellm.ai/docs/proxy/deploy
Docker Quick Start: https://docs.litellm.ai/docs/proxy/docker_quick_start

name: litellm
services:
  litellm:
    image: ghcr.io/berriai/litellm:v1.80.0-stable
    command:
      - --config=/app/config.yaml
      - --telemetry=false
    container_name: litellm
    restart: unless-stopped
    volumes:
      - ./litellm/config.yaml:/app/config.yaml
    networks:
      - proxy
    environment:
      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-""}
      OPENAI_API_KEY: ${OPENAI_API_KEY:-""}
      GEMINI_API_KEY: ${GEMINI_API_KEY:-""}
      OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-""}
      LOCAL_OLLAMA_API_BASE: https://ollama.${MYDOMAIN}
      REMOTE_OLLAMA_API_BASE: http://localhost:11444
    healthcheck:  # Defines the health check configuration for the container
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1"]  # Command to execute for health check
      interval: 30s  # Perform health check every 30 seconds
      timeout: 10s   # Health check command times out after 10 seconds
      retries: 3     # Retry up to 3 times if health check fails
      start_period: 40s  # Wait 40 seconds after container start before beginning health checks
    labels:
      traefik.enable: true
      traefik.http.routers.litellm.middlewares: localaccess@file
      traefik.http.services.litellm.loadbalancer.server.port: 4000
      homepage.group: AI
      homepage.name: "LiteLLM"
      homepage.icon: chatgpt.png
      homepage.href: https://litellm.${MYDOMAIN}/
      homepage.description: "LLM Gateway to call 100+ LLM APIs in OpenAI format"

networks:
  proxy:
    external: true