⁨Xinference⁩: تشغيل ⁨Llama 3⁩ و⁨Qwen⁩ و⁨ChatGLM⁩ و⁨Mistral⁩ محليًا ⁨2026⁩

# Instalación completa — backends CUDA + transformers + llama.cpp
pip install "xinference[all]"

# Instalación solo CPU (sin GPU)
pip install xinference

# Iniciar el servidor local (interfaz web en http://localhost:9997)
xinference-local

# O vincular a un host específico para acceso por LAN
xinference-local --host 0.0.0.0 --port 9997

# Llama 3.3 8B Instruct (GPU, backend transformers)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine transformers \
  --model-size-in-billions 8

# Llama 3.3 8B Instruct (CPU, cuantización Q4_K_M)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine llama.cpp \
  --model-size-in-billions 8 \
  --quantization q4_k_m

# Qwen 3 7B Instruct (GPU)
xinference launch \
  --model-name qwen2.5-instruct \
  --model-engine transformers \
  --model-size-in-billions 7

# ChatGLM4 9B (GPU)
xinference launch \
  --model-name chatglm4 \
  --model-engine transformers \
  --model-size-in-billions 9

# Mistral Small Instruct v0.3 (GPU)
xinference launch \
  --model-name mistral-instruct-v0.3 \
  --model-engine transformers \
  --model-size-in-billions 7

# Mixtral 8x22B Instruct (backend vLLM, requiere 26+ GB VRAM)
xinference launch \
  --model-name mixtral-instruct-v0.1 \
  --model-engine vllm \
  --model-size-in-billions 46

from openai import OpenAI

client = OpenAI(
    api_key="not-required",   # Xinference accepts any non-empty string
    base_url="http://localhost:9997/v1"
)

# Chat completion — works for Llama 3, Qwen, ChatGLM, Mistral
response = client.chat.completions.create(
    model="llama-3.1-instruct",   # use the model name as the UID
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Summarise the GDPR in 3 bullet points."}
    ]
)
print(response.choices[0].message.content)

# Embedding model (run a separate xinference launch for bge-base-en-v1.5 first)
embedding = client.embeddings.create(
    model="bge-base-en-v1.5",
    input="Local LLMs preserve data privacy."
)
print(embedding.data[0].embedding[:5])