llama-swap README for basic config, this covers:

• llama-server from llama.cpp
• or ik_llama.cpp
• whisper-server
• sd-server

Config example for docker-based backends

So in podman with GPU access or docker it's possible to run something like:

• https://github.com/remsky/Kokoro-FastAPI
• https://github.com/Shadowfita/parakeet-tdt-0.6b-v2-fastapi
• https://github.com/Lex-au/Orpheus-FastAPI

Config example (with podman, docker should be similar):

macros:
  "modeldir": ./models
  "bindir": ./bin
  "comfymodeldir": ${modeldir}/comfyui

  "llama-server": >
    ${bindir}/llama-server
    --port ${PORT}
    --jinja

  "moss-tts-server": ${bindir}/moss-tts-server

  "ikllama-server": >
    ${bindir}/ik_llama-server
    --port ${PORT}
    --jinja

  "sd-server": >
    ${bindir}/sd-server
    --listen-port ${PORT}
    --diffusion-fa
    --verbose

  "whisper-server": ${bindir}/whisper-server

models:
  "whisper":
    name: "Whisper Audio Transcription"
    proxy: "http://127.0.0.1:${PORT}"
    checkEndpoint: /v1/audio/transcriptions/
    cmd: |
      ${whisper-server}
      --host 127.0.0.1
      --port ${PORT}
      --model ${modeldir}/ggml-large-v3-turbo-q8_0.bin
      --request-path /v1/audio/transcriptions
      --inference-path ""

  "z-image-turbo":
    name: "Z-Image Turbo text-to-image"
    checkEndpoint: /
    cmd: |
      ${sd-server}
      --diffusion-model ${modeldir}/z-image-turbo-Q8_0.gguf
      --vae ${comfymodeldir}/vae/ae.safetensors
      --llm ${modeldir}/Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf
      --lora-model-dir ${comfymodeldir}/loras/
      --steps 8
      --cfg-scale 1.0
      --vae-conv-direct
      --sampling-method euler
      --scheduler sgm_uniform
      --seed -1

  "openmoss-1.5-q8":
    name: "MOSS-TTS-v1.5"
    proxy: "http://127.0.0.1:${PORT}"
    #env:
    #  - "LD_LIBRARY_PATH=${env.HOME}/src/llama.cpp/bin"
    cmd: |
      ${moss-tts-server}
      --host 127.0.0.1
      --port ${PORT}
      --no-webui
      --model ${modeldir}/moss-tts-1.5-q8_0.gguf

  "kokoro-tts":
    name: "kokoro TTS"
    proxy: "http://127.0.0.1:${PORT}"
    useModelName: "tts-1"
    checkEndpoint: /health
    cmd: |
      podman run
      --rm
      --replace
      --name ${MODEL_ID}
      -p ${PORT}:8880
      --device nvidia.com/gpu=all
      --env 'API_LOG_LEVEL=INFO'
      ghcr.io/remsky/kokoro-fastapi-gpu:latest
    cmdStop: podman stop ${MODEL_ID}

  "parakeet-tdt":
    name: "parakeet-tdt-0.6b-v3 ASR"
    proxy: "http://127.0.0.1:${PORT}"
    checkEndpoint: /health
    cmd: |
      podman run
      --rm
      --replace
      --name ${MODEL_ID}
      -p ${PORT}:5092
      --device nvidia.com/gpu=all
      localhost/parakeet-tdt:gpu
    cmdStop: podman stop ${MODEL_ID}

  "qwen3.6-27b-q4-novision":
    name: "Qwen 3.6 27B Q4 text only"
    filters:
      stripParams: "temperature, top_k, top_p, min_p, repeat_penalty, presence_penalty"
      setParams:
        # Default: reasoning-coding (precise coding with thinking enabled)
        chat_template_kwargs:
          enable_thinking: true
        temperature: 0.6
        presence_penalty: 0.0
      setParamsByID:
        "${MODEL_ID}:reasoning-coding":
          chat_template_kwargs:
            enable_thinking: true
          temperature: 0.6
          presence_penalty: 0.0
        "${MODEL_ID}:reasoning-general":
          chat_template_kwargs:
            enable_thinking: true
          temperature: 1.0
          presence_penalty: 1.5
        "${MODEL_ID}:instruct-general":
          chat_template_kwargs:
            enable_thinking: false
          temperature: 0.7
          top_p: 0.8
          presence_penalty: 1.5
        "${MODEL_ID}:instruct-reasoning":
          chat_template_kwargs:
            enable_thinking: false
          temperature: 1.0
          top_p: 1.0
          presence_penalty: 1.5
    cmd: |
      ${llama-server}
      --cache-type-k q8_0
      --cache-type-v q8_0
      --flash-attn on
      --n-gpu-layers 999
      --spec-type ngram-mod
      --parallel 1
      --top-p 0.95
      --top-k 20
      --min-p 0.0
      --repeat-penalty 1.0
      --reasoning on
      --chat-template-kwargs '{"preserve_thinking": true}'
      --ctx-size 131072
      --fit off
      #--no-mmproj-offload
      #--image-min-tokens 256 --image-max-tokens 768
      #--mmproj ${modeldir}/mmproj-Qwen_Qwen3.6-27B-bf16.gguf
      --model ${modeldir}/Qwen_Qwen3.6-27B-Q4_K_M.gguf