How to Run Kimi K2.5 Locally on Ubuntu

# Install Ollama on Ubuntu
curl -fsSL https://ollama.com/install.sh | sh

# Verify installation
ollama --version

# Pull the cloud-backed model
ollama pull kimi-k2.5:cloud

# Run in interactive chat mode
ollama run kimi-k2.5:cloud

# Run with a single prompt
ollama run kimi-k2.5:cloud "Explain the difference between LoRA and QLoRA"

# Run with a system prompt
ollama run kimi-k2.5:cloud --system "You are a senior Python developer" \
  "Write a FastAPI endpoint for file upload"

# Ollama exposes an OpenAI-compatible API on port 11434
curl http://localhost:11434/api/generate -d '{
  "model": "kimi-k2.5:cloud",
  "prompt": "Write a bash script to monitor GPU temperature",
  "stream": false,
  "options": {
    "temperature": 0.7,
    "num_ctx": 65536
  }
}'

sudo apt-get update
sudo apt-get install -y pciutils build-essential cmake curl libcurl4-openssl-dev git

git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp

# Build with CUDA (requires NVIDIA GPU + CUDA toolkit)
cmake -B build \
  -DBUILD_SHARED_LIBS=OFF \
  -DGGML_CUDA=ON

cmake --build build --config Release -j$(nproc) --clean-first \
  --target llama-cli llama-server llama-gguf-split

# Copy binaries to the llama.cpp root for convenience
cp build/bin/llama-* .

# For CPU-only builds, replace -DGGML_CUDA=ON with -DGGML_CUDA=OFF

pip install -U huggingface_hub

# Download the 1.8-bit quant (240 GB, minimum size)
huggingface-cli download unsloth/Kimi-K2.5-GGUF \
  --local-dir Kimi-K2.5-GGUF \
  --include "*UD-TQ1_0*"

# Or download the 2-bit XL quant (375 GB, recommended)
huggingface-cli download unsloth/Kimi-K2.5-GGUF \
  --local-dir Kimi-K2.5-GGUF \
  --include "*UD-Q2_K_XL*"

# Or download the 4-bit quant (621 GB, near-lossless)
huggingface-cli download unsloth/Kimi-K2.5-GGUF \
  --local-dir Kimi-K2.5-GGUF \
  --include "*Q4_K_M*"

export LLAMA_CACHE="Kimi-K2.5-GGUF"

LLAMA_SET_ROWS=1 ./llama-cli \
  --model Kimi-K2.5-GGUF/UD-TQ1_0/Kimi-K2.5-UD-TQ1_0-00001-of-00005.gguf \
  --temp 1.0 \
  --min-p 0.01 \
  --top-p 0.95 \
  --ctx-size 16384 \
  --seed 3407

LLAMA_SET_ROWS=1 ./llama-server \
  --model Kimi-K2.5-GGUF/UD-TQ1_0/Kimi-K2.5-UD-TQ1_0-00001-of-00005.gguf \
  --special \
  --alias "kimi-k2.5" \
  --min_p 0.01 \
  --ctx-size 16384 \
  --port 8001 \
  --host 0.0.0.0 \
  --kv-unified

# Server starts at http://0.0.0.0:8001
# OpenAI-compatible endpoint: http://localhost:8001/v1/chat/completions

# Offload all MoE expert layers to CPU (minimum VRAM usage)
LLAMA_SET_ROWS=1 ./llama-server \
  --model Kimi-K2.5-GGUF/UD-TQ1_0/Kimi-K2.5-UD-TQ1_0-00001-of-00005.gguf \
  -ot ".ffn_.*_exps.=CPU" \
  --ctx-size 16384 \
  --port 8001

# Offload only up/down projection MoE (keep gate in VRAM, faster)
LLAMA_SET_ROWS=1 ./llama-server \
  --model Kimi-K2.5-GGUF/UD-TQ1_0/Kimi-K2.5-UD-TQ1_0-00001-of-00005.gguf \
  -ot ".ffn_(up|down)_exps.=CPU" \
  --ctx-size 16384 \
  --port 8001

# Auto-fit: let llama.cpp decide optimal layer distribution
LLAMA_SET_ROWS=1 ./llama-server \
  --model Kimi-K2.5-GGUF/UD-TQ1_0/Kimi-K2.5-UD-TQ1_0-00001-of-00005.gguf \
  --fit on \
  --ctx-size 16384 \
  --port 8001

pip install uv
uv pip install -U vllm --pre \
  --extra-index-url https://wheels.vllm.ai/nightly/cu129

vllm serve moonshotai/Kimi-K2.5 \
  -tp 8 \
  --mm-encoder-tp-mode data \
  --compilation_config.pass_config.fuse_allreduce_rms true \
  --tool-call-parser kimi_k2 \
  --reasoning-parser kimi_k2 \
  --enable-auto-tool-choice \
  --trust-remote-code

vllm bench serve \
  --backend openai-chat \
  --endpoint /v1/chat/completions \
  --model moonshotai/Kimi-K2.5 \
  --dataset-name hf \
  --dataset-path lmarena-ai/VisionArena-Chat \
  --num-prompts 1000 \
  --request-rate 20

from openai import OpenAI

# Connect to local llama.cpp server (port 8001)
# Or Ollama (port 11434) or vLLM (port 8000)
client = OpenAI(
    base_url="http://127.0.0.1:8001/v1",
    api_key="sk-no-key-required",
)

response = client.chat.completions.create(
    model="kimi-k2.5",
    messages=[
        {"role": "system", "content": "You are a helpful coding assistant."},
        {"role": "user", "content": "Write a Python function to detect GPU memory leaks"},
    ],
    temperature=0.7,
    max_tokens=2048,
)
print(response.choices[0].message.content)

import requests
import json

url = "http://localhost:11434/api/generate"
payload = {
    "model": "kimi-k2.5:cloud",
    "prompt": "Explain CUDA unified memory in detail",
    "stream": True,
    "options": {"temperature": 0.7, "num_ctx": 32768}
}

response = requests.post(url, json=payload, stream=True)
for line in response.iter_lines():
    if line:
        data = json.loads(line)
        print(data.get("response", ""), end="", flush=True)

curl http://localhost:8001/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "kimi-k2.5",
    "messages": [
      {"role": "user", "content": "How does MoE routing work in transformer models?"}
    ],
    "temperature": 0.7,
    "max_tokens": 1024
  }'

# Speed boost for MoE models
export LLAMA_SET_ROWS=1

# Set model cache directory
export LLAMA_CACHE="/path/to/models"

# Expose all 8 GPUs to Ollama
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7

# Allow 4 parallel requests
export OLLAMA_NUM_PARALLEL=4

# Keep only 1 model loaded (saves VRAM)
export OLLAMA_MAX_LOADED_MODELS=1

# Restart Ollama with new settings
sudo systemctl restart ollama

# docker-compose.yml
version: '3.8'
services:
  ollama:
    image: ollama/ollama:latest
    volumes:
      - ollama_data:/root/.ollama
    ports:
      - '11434:11434'
    environment:
      - OLLAMA_NUM_PARALLEL=4
      - CUDA_VISIBLE_DEVICES=0,1,2,3
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 4
              capabilities: [gpu]

volumes:
  ollama_data:

# Install Xet for faster Hugging Face downloads
pip install huggingface_hub[hf_xet]

# Resume interrupted downloads
huggingface-cli download unsloth/Kimi-K2.5-GGUF \
  --local-dir Kimi-K2.5-GGUF \
  --include "*UD-TQ1_0*" \
  --resume-download

# Verify CUDA installation
nvidia-smi
nvcc --version

# If nvcc is missing, install CUDA toolkit
sudo apt-get install -y nvidia-cuda-toolkit

# Rebuild llama.cpp with CUDA
cd llama.cpp
cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON
cmake --build build --config Release -j$(nproc) --clean-first

# Create an 8x H100 instance for full BF16 vLLM deployment
curl -X POST https://api.gpulabs.cloud/gpu/instances \
  -H "Authorization: Bearer sk_your_api_key" \
  -H "Content-Type: application/json" \
  -d '{
    "offerId": "h100-sxm-80gb-8x",
    "sshKeyId": "your-ssh-key-id",
    "name": "kimi-k25-production"
  }'

# SSH into the instance
ssh root@<instance-ip>

# Follow Method 3 (vLLM) from this guide
pip install uv
uv pip install -U vllm --pre --extra-index-url https://wheels.vllm.ai/nightly/cu129
vllm serve moonshotai/Kimi-K2.5 -tp 8 --trust-remote-code

# Create a single A100 80GB instance
curl -X POST https://api.gpulabs.cloud/gpu/instances \
  -H "Authorization: Bearer sk_your_api_key" \
  -H "Content-Type: application/json" \
  -d '{
    "offerId": "a100-80gb-sxm",
    "sshKeyId": "your-ssh-key-id",
    "name": "kimi-k25-gguf"
  }'

# SSH in and run llama.cpp with 1.8-bit quant
# Follow Method 2 steps 1-5 from this guide