Xinference: Llama 3, Qwen, ChatGLM & Mistral lokal ausführen 2026

# Vollständige Installation — CUDA + transformers + llama.cpp Backends
pip install "xinference[all]"

# Nur-CPU-Installation (kein GPU erforderlich)
pip install xinference

# Lokalen Server starten (Web-Oberfläche auf http://localhost:9997)
xinference-local

# Oder an bestimmten Host für LAN-Zugriff binden
xinference-local --host 0.0.0.0 --port 9997

# Llama 3.1 8B Instruct (GPU, transformers backend)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine transformers \
  --model-size-in-billions 8

# Llama 3.1 8B Instruct (CPU, Q4_K_M quantization)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine llama.cpp \
  --model-size-in-billions 8 \
  --quantization q4_k_m

# Qwen 2.5 7B Instruct (GPU)
xinference launch \
  --model-name qwen2.5-instruct \
  --model-engine transformers \
  --model-size-in-billions 7

# ChatGLM4 9B (GPU)
xinference launch \
  --model-name chatglm4 \
  --model-engine transformers \
  --model-size-in-billions 9

# Mistral 7B Instruct v0.3 (GPU)
xinference launch \
  --model-name mistral-instruct-v0.3 \
  --model-engine transformers \
  --model-size-in-billions 7

# Mixtral 8x7B Instruct (vLLM backend, requires 26+ GB VRAM)
xinference launch \
  --model-name mixtral-instruct-v0.1 \
  --model-engine vllm \
  --model-size-in-billions 46

from openai import OpenAI

client = OpenAI(
    api_key="not-required",   # Xinference accepts any non-empty string
    base_url="http://localhost:9997/v1"
)

# Chat completion — works for Llama 3, Qwen, ChatGLM, Mistral
response = client.chat.completions.create(
    model="llama-3.1-instruct",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Summarise the GDPR in 3 bullet points."}
    ]
)
print(response.choices[0].message.content)

# Embedding model (run a separate xinference launch for bge-base-en-v1.5 first)
embedding = client.embeddings.create(
    model="bge-base-en-v1.5",
    input="Local LLMs preserve data privacy."
)
print(embedding.data[0].embedding[:5])