Xinference: Llama 3, Qwen, ChatGLM & Mistral 로컬 실행 가이드 2026

# 전체 설치 — CUDA + transformers + llama.cpp 백엔드
pip install "xinference[all]"

# CPU 전용 설치 (GPU 불필요)
pip install xinference

# 로컬 서버 시작 (웹 UI: http://localhost:9997)
xinference-local

# LAN 접근을 위한 특정 호스트 바인딩
xinference-local --host 0.0.0.0 --port 9997

# Llama 3.3 8B Instruct (GPU, transformers 백엔드)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine transformers \
  --model-size-in-billions 8

# Llama 3.3 8B Instruct (CPU, Q4_K_M 양자화)
xinference launch \
  --model-name llama-3.1-instruct \
  --model-engine llama.cpp \
  --model-size-in-billions 8 \
  --quantization q4_k_m

# Qwen 3 7B Instruct (GPU)
xinference launch \
  --model-name qwen2.5-instruct \
  --model-engine transformers \
  --model-size-in-billions 7

# ChatGLM4 9B (GPU)
xinference launch \
  --model-name chatglm4 \
  --model-engine transformers \
  --model-size-in-billions 9

# Mistral Small Instruct v0.3 (GPU)
xinference launch \
  --model-name mistral-instruct-v0.3 \
  --model-engine transformers \
  --model-size-in-billions 7

# Mixtral 8x22B Instruct (vLLM 백엔드, 26 GB 이상 VRAM 필요)
xinference launch \
  --model-name mixtral-instruct-v0.1 \
  --model-engine vllm \
  --model-size-in-billions 46

from openai import OpenAI

client = OpenAI(
    api_key="not-required",   # Xinference는 빈 문자열이 아닌 모든 값을 수락합니다
    base_url="http://localhost:9997/v1"
)

# 채팅 완성 — Llama 3, Qwen, ChatGLM, Mistral 모두 동작합니다
response = client.chat.completions.create(
    model="llama-3.1-instruct",   # 모델 이름을 UID로 사용합니다
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Summarise the GDPR in 3 bullet points."}
    ]
)
print(response.choices[0].message.content)

# 임베딩 모델 (먼저 bge-base-en-v1.5를 별도로 xinference launch 하십시오)
embedding = client.embeddings.create(
    model="bge-base-en-v1.5",
    input="Local LLMs preserve data privacy."
)
print(embedding.data[0].embedding[:5])