Sorry-

Using LiteLLM to route to models on different inference engines (like vLLM and llama.cpp): https://github.com/BerriAI/litellm

LiteLLM provides metrics to Prometheus, in which those metrics can be pulled into Grafana for the dashboard: https://github.com/grafana/grafana

My docker-compose stack for this setup:

services:
  litellm-db:
    image: postgres:15
    container_name: litellm-db
    restart: unless-stopped
    environment:
      - POSTGRES_DB=litellm
      - POSTGRES_USER=litellm
      - POSTGRES_PASSWORD=litellm_secure_pass
    volumes:
      - /mnt/AI/litellm/db_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U litellm"]
      interval: 5s
      timeout: 5s
      retries: 5
    networks:
      - ai-net

  litellm-proxy:
    image: ghcr.io/berriai/litellm:main-latest
    container_name: litellm
    restart: unless-stopped
    ports:
      - "8484:4000" 
    volumes:
      - /mnt/AI/litellm/config.yaml:/app/config.yaml:ro
    environment:
      - OPENAI_API_KEY=litellm
      - USE_PRISMA_MIGRATE="True"
      - LITELLM_MASTER_KEY="sk-yourkey"
      - LITELLM_SALT_KEY="sk-salt-yoursaltkey"
      - UI_USERNAME=admin   
      - UI_PASSWORD=password
      - DATABASE_URL=postgresql://litellm:litellm_secure_pass@litellm-db:5432/litellm
      - STORE_MODEL_IN_DB="True"
    command: ["--config", "/app/config.yaml", "--port", "4000"]
    depends_on:
      litellm-db:
        condition: service_healthy
    networks:
      - ai-net

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - /mnt/AI/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - /mnt/AI/prometheus/data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
    networks:
      - ai-net

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3007:3000"
    volumes:
      - /mnt/AI/grafana/data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin # Change this on first login
    networks:
      - ai-net

networks:
  ai-net:
    external: true