dual 3090 with nvlink. I get 30-150tok/s with kv cache and model quantization, 3090 has INT4 accelerators and speculative decode 5 step is the speed boost, depending on cache hit.

services:
  vllm:
    image: vllm/vllm-openai:latest-cu130
    container_name: vllm    
    env_file:
      - .env
    restart: unless-stopped
    # ports:
    #   - "8999:8000"
    volumes:
      - /home/jkim/llama.cpp_files/.cache/huggingface:/root/.cache/huggingface
    environment:
      # - VLLM_LOGGING_LEVEL=DEBUG
      # - VLLM_LOG_STATS_INTERVAL=1
      # - NCCL_DEBUG=TRACE
      # - VLLM_TRACE_FUNCTION=1
      # - NCCL_IGNORE_DISABLED_P2P=1
      # - CUDA_LAUNCH_BLOCKING=1
      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
      - CUDA_VISIBLE_DEVICES=0,1
      - RAY_memory_monitor_refresh_ms=0
      - NCCL_CUMEM_ENABLE=0
      # - VLLM_SLEEP_WHEN_IDLE=1
      - VLLM_ENABLE_CUDAGRAPH_GC=1
      - VLLM_USE_FLASHINFER_SAMPLER=1
      # - VLLM_SERVER_DEV_MODE=1 #       --enable-sleep-mode
      - OMP_NUM_THREADS=1
    shm_size: 4g
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 2
              capabilities: [gpu]
    command: >
      cyankiwi/Qwen3.6-27B-AWQ-BF16-INT4
      --kv-cache-dtype fp8
      --tensor-parallel-size 2
      --gpu-memory-utilization 0.90
      --max-model-len 262144
      --quantization compressed-tensors
      --max-num-seqs 16
      --block-size 32
      --max-num-batched-tokens 4096
      --enable-prefix-caching
      --chat-template /root/.cache/huggingface/qwen3.5-enhanced.jinja
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --reasoning-parser qwen3
      --attention-backend FLASHINFER
      --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
      --compilation-config '{"cudagraph_mode": "PIECEWISE"}'
      --use-tqdm-on-load
      -O3


    networks:
      - reverse-proxy-net


networks:
  reverse-proxy-net:
    name: reverse-proxy-net
    external: true