I'm using llama.cpp docker with vulkan. I tuned it so it fits my VRAM with desktop and about 1-3 GB left. I have 128GB of DDR4-3200 CL16 (Kingston KF3200C16D4/32GX) and a 5950X on a Gigabyte X570S. GPU is a Sapphire Nitro+ 7900XTX with 24GB.

Qwen 3.5 27B Q5_K_M

[Service]
Restart=always
RestartSec=10

Environment=HOST=127.0.0.1
Environment=PORT=8080
Environment=MODELBASE=/home/beda/workspace/widgets/fancyllm/models
Environment=MODEL=unsloth/Qwen3.5-27B-Q5_K_M.gguf
Environment=IMAGE=ghcr.io/ggml-org/llama.cpp:server-vulkan

ExecStartPre=-/usr/bin/podman pull ${IMAGE}

ExecStart=/usr/bin/podman run \
--name llama-server \
--replace \
-p ${HOST}:${PORT}:${PORT} \
-v ${MODELBASE}:/models:Z \
--device /dev/dri \
--group-add video \
${IMAGE} \
-m /models/${MODEL} \
--host 0.0.0.0 --port ${PORT} \
--n-gpu-layers 999 \
-c 32768 \
--batch-size 256 \
--ubatch-size 256 \
--parallel 1 \
--threads 12 \
--threads-batch 12 \
--no-context-shift \
--reasoning-format none

ExecStop=/usr/bin/podman stop -t 10 llama-server

Qwen 3.5 122B MoE Q4_K_M

[Service]
Restart=always
RestartSec=10

Environment=HOST=127.0.0.1
Environment=PORT=8080
Environment=MODELBASE=/home/beda/workspace/widgets/fancyllm/models
Environment=MODEL=unsloth/Qwen3.5-122B-A10B-Q4_K_M.gguf
Environment=IMAGE=ghcr.io/ggml-org/llama.cpp:server-vulkan

ExecStartPre=-/usr/bin/podman pull ${IMAGE}

ExecStart=/usr/bin/podman run \
--name llama-server \
--replace \
-p ${HOST}:${PORT}:${PORT} \
-v ${MODELBASE}:/models:Z \
--device /dev/dri \
--group-add video \
${IMAGE} \
-m /models/${MODEL} \
--host 0.0.0.0 --port ${PORT} \
--n-gpu-layers 14 \
--n-cpu-moe 12 \
-c 16384 \
--batch-size 256 \
--parallel 1 \
--threads 16 \
--threads-batch 16 \
--no-context-shift \
--no-mmap \
--reasoning-format none

ExecStop=/usr/bin/podman stop -t 10 llama-server