Initial commit

.env.example

@@ -0,0 +1,6 @@
+QDRANT_API_KEY=somerandomstring
+QDRANT_READ_ONLY_API_KEY=anotherrandomstring
+MAX_JOBS=12
+TG_MODEL=Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4
+EMBEDDING_MODEL=Qwen/Qwen3-Embedding-0.6B
+VLLM_CHAT_URL_BASE=http://localhost:8001

.gitignore

@@ -0,0 +1,2 @@
+**/__pycache__
+venv

README.md

@@ -0,0 +1,62 @@
+# Text Generation and Embedding API/Router
+
+FastAPI server that routes LLM chat requests to vLLM and provides local text embeddings.
+
+## Quick Start
+
+### Prerequisites
+- Python 3.8+ with CUDA support
+- [vLLM](https://github.com/vllm-project/vllm) 
+- [PM2](https://pm2.keymetrics.io/) process manager
+
+### Installation
+```bash
+pip install -r requirements.txt
+```
+
+### Running
+1. Start vLLM server:
+```bash
+pm2 start --name vllm 'vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 --dtype half --quantization gptq --gpu-memory-utilization 0.7 --port 8001'
+```
+
+2. Start the router:
+```bash
+pm2 start --name llm_proxy 'uvicorn app.main:app --host 0.0.0.0 --port 4444'
+```
+
+### Environment Variables
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `TG_MODEL` | Text generation model | `Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4` |
+| `EMBEDDING_MODEL` | Embedding model | `Qwen/Qwen3-Embedding-0.6B` |
+| `VLLM_CHAT_URL_BASE` | vLLM server base URL | `http://localhost:8001` |
+| `MAX_EMBED_TEXT_SIZE` | Max text length for embeddings | `16384` |
+
+## API Endpoints
+
+- `POST /health` - Service health check
+- `POST /v1/chat/completions` - OpenAI-compatible chat completions (proxied to vLLM)
+- `POST /v1/embeddings` - Text embeddings (local inference)
+
+### Example Usage
+```bash
+# Chat completion
+curl localhost:4444/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"Hello!"}]}'
+
+# Text embeddings
+curl localhost:4444/v1/embeddings \
+  -H 'Content-Type: application/json' \
+  -d '{"texts":["Hello world"]}'
+```
+
+## Deployment
+
+For production deployment with all dependencies:
+```bash
+export QDRANT_API_KEY="your_secure_key"
+export QDRANT_READ_ONLY_API_KEY="your_readonly_key"
+./scripts/deploy.sh
+```

app/config.py

@@ -0,0 +1,11 @@
+import os
+
+DEFAULT_TG_MODEL="Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4"
+DEFAULT_EMBEDDING_MODEL="Qwen/Qwen3-Embedding-0.6B"
+DEFAULT_VLLM_CHAT_URL_BASE="http://localhost:8001"
+DEFAULT_MAX_EMBED_TEXT_SIZE=16384
+
+TG_MODEL = os.getenv("TG_MODEL", DEFAULT_TG_MODEL)
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
+VLLM_CHAT_URL_BASE = os.getenv("VLLM_CHAT_URL_BASE", DEFAULT_VLLM_CHAT_URL_BASE)
+MAX_EMBED_TEXT_SIZE=os.getenv("MAX_EMBED_TEXT_SIZE", DEFAULT_MAX_EMBED_TEXT_SIZE)

app/main.py

@@ -0,0 +1,19 @@
+from fastapi import FastAPI
+from contextlib import asynccontextmanager
+from app.routes import chat, embeddings, health
+from app.utils.logging import logger
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    try:
+        yield
+    finally:
+        logger.info("Application shutdown")
+
+
+app = FastAPI(title="vLLM API", version="0.1.0", lifespan=lifespan)
+
+# Include routers
+app.include_router(health.router, prefix="/health", tags=["health"])
+app.include_router(chat.router, prefix="/v1/chat", tags=["chat"])
+app.include_router(embeddings.router, prefix="/v1/embeddings", tags=["embeddings"])

app/models/embedding.py

@@ -0,0 +1,26 @@
+from typing import List
+import torch
+from transformers import AutoModel, AutoTokenizer
+from app.utils.logging import logger
+from app.config import EMBEDDING_MODEL
+
+class EmbeddingModel:
+    def __init__(self):
+        logger.info(f"Loading embedding model: {EMBEDDING_MODEL}")
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(EMBEDDING_MODEL)
+            self.model = AutoModel.from_pretrained(EMBEDDING_MODEL).to("cuda")
+        except Exception as e:
+            raise RuntimeError(f"Could not load embedding model: {e}")
+
+
+    def encode(self, texts: List[str]) -> List[List[float]]:
+        if not self.tokenizer or not self.model:
+            raise RuntimeError(f"Embedding model is not available")
+        inputs = self.tokenizer(texts, padding=True, return_tensors="pt").to("cuda")
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        embeddings = outputs.last_hidden_state.mean(dim=1)  # Mean pooling
+        return embeddings.cpu().numpy().tolist()
+    
+model = EmbeddingModel()

app/models/llm.py

@@ -0,0 +1,27 @@
+from vllm.engine.async_llm_engine import AsyncLLMEngine
+from vllm.engine.arg_utils import AsyncEngineArgs
+from typing import Optional
+from app.config import TG_MODEL
+import os
+
+class VLLMEngine:
+    model_name: Optional[str] = None
+    engine: Optional[AsyncLLMEngine] = None
+
+    @classmethod
+    async def init_engine(cls):
+        """Initialize vLLM engine with quantization."""
+        cls.model_name = TG_MODEL
+        cls.engine = AsyncLLMEngine.from_engine_args(
+            AsyncEngineArgs(
+                model=cls.model_name,
+                quantization="gptq",  # awq or "gptq"
+                tensor_parallel_size=1,  # Adjust for multi-GPU
+                max_num_seqs=100,       # Concurrent requests
+                gpu_memory_utilization=0.7, # Total allocation of the VRAM
+                dtype="half"
+            )
+        )
+
+# Singleton instance
+vllm_engine = VLLMEngine()

app/routes/chat.py

@@ -0,0 +1,31 @@
+import os
+import httpx
+
+from fastapi import APIRouter, Request, HTTPException
+from fastapi.responses import StreamingResponse, JSONResponse
+from app.config import VLLM_CHAT_URL_BASE
+from app.utils.logging import logger
+
+router = APIRouter()
+
+@router.post("/completions")
+async def chat_completion(request: Request):
+    body = await request.body()
+    try:
+        async with httpx.AsyncClient(timeout=20.0) as client:
+            async with client.stream("POST", f"{VLLM_CHAT_URL_BASE}/v1/chat/completions", content=body, headers={'Content-type':'application/json'}) as resp:
+                if resp.headers.get("content-type") == "text/event-stream":
+                    return StreamingResponse(resp.aiter_raw(), media_type="text/event-stream")
+                else:
+                    data = await resp.aread()
+                    return JSONResponse(content=data.decode(), status_code=resp.status_code)
+    except httpx.ConnectError:
+        logger.error(f"Unable to connect to vLLM server at {VLLM_CHAT_URL_BASE}")
+        raise HTTPException(status_code=503, detail="LLM service temporarily unavailable")
+    except httpx.TimeoutException:
+        logger.error("Request to vLLM server timed out")
+        raise HTTPException(status_code=504, detail="LLM service request timeout")
+    except Exception as e:
+        logger.error(f"Unexpected error in chat completion: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+

app/routes/embeddings.py

@@ -0,0 +1,11 @@
+from fastapi import APIRouter
+from app.schemas.embeddings import EmbeddingRequest
+from app.models.embedding import model
+
+router = APIRouter()
+
+@router.post("")
+async def create_embedding(request: EmbeddingRequest):
+    """Generate embeddings for input text."""
+    embeddings = model.encode(request.texts)
+    return {"embeddings": embeddings}

app/routes/health.py

@@ -0,0 +1,25 @@
+import httpx
+from fastapi import APIRouter
+from app.config import VLLM_CHAT_URL_BASE
+from app.utils.logging import logger
+
+router = APIRouter()
+
+@router.post("")
+async def health():
+    """Get service health and check dependent services"""
+    health_status = {"status": "healthy", "services": {
+        "embeddings": "healthy"
+    }}
+    
+    # Check vLLM server
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            response = await client.get(f"{VLLM_CHAT_URL_BASE}/health")
+            health_status["services"]["vllm"] = "healthy" if response.status_code == 200 else "unhealthy"
+    except Exception as e:
+        logger.warning(f"vLLM health check failed: {e}")
+        health_status["services"]["vllm"] = "unhealthy"
+        health_status["status"] = "degraded" 
+    
+    return health_status

app/schemas/chat.py

@@ -0,0 +1,38 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from vllm import SamplingParams
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+
+class ChatRequest(BaseModel):
+    messages: List[ChatMessage]
+    temperature: float = 0.7
+    top_p: float = 1.0
+    top_k: int = -1
+    max_tokens: int = 512
+    stop: Optional[List[str]] = None
+    presence_penalty: float = 0.0
+    frequency_penalty: float = 0.0
+    repetition_penalty: float = 1.0
+    n: int = 1
+    best_of: int = 1
+    seed: Optional[int] = None
+    logprobs: Optional[int] = Field(default=None, ge=1)
+
+    def to_sampling_params(self) -> SamplingParams:
+        return SamplingParams(
+            temperature=self.temperature,
+            top_p=self.top_p,
+            top_k=self.top_k,
+            max_tokens=self.max_tokens,
+            stop=self.stop,
+            presence_penalty=self.presence_penalty,
+            frequency_penalty=self.frequency_penalty,
+            repetition_penalty=self.repetition_penalty,
+            n=self.n,
+            best_of=self.best_of,
+            seed=self.seed,
+            logprobs=self.logprobs,
+        )

app/schemas/embeddings.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field, field_validator
+from typing import List
+from app.config import MAX_EMBED_TEXT_SIZE
+
+class EmbeddingRequest(BaseModel):
+    texts: List[str] = Field(..., min_items=1, max_items=100, description="List of texts to embed")
+    
+    @field_validator('texts')
+    def validate_texts(cls, v):
+        for text in v:
+            if not text.strip():
+                raise ValueError("Empty texts are not allowed")
+            if len(text) > MAX_EMBED_TEXT_SIZE: # Limit texts to embed
+                raise ValueError(f"Text too long (max {MAX_EMBED_TEXT_SIZE} characters)")
+        return v

app/utils/logging.py

@@ -0,0 +1,14 @@
+import logging
+import sys
+
+def setup_logging():
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.StreamHandler(sys.stdout)
+        ]
+    )
+    return logging.getLogger(__name__)
+
+logger = setup_logging()

pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "vllm_api"
+version = "0.1.0"
+
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -0,0 +1,7 @@
+fastapi
+uvicorn
+pydantic
+transformers
+httpx
+## BUILD MANUALLY
+# vllm

scripts/deploy.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+set -e  # Exit on error
+
+# Validate Qdrant API keys
+if [ -z "$QDRANT_API_KEY" ]; then
+    echo "ERROR: QDRANT_API_KEY environment variable is required"
+    echo "Please set environment variables before running deployment"
+    exit 1
+fi
+
+if [ -z "$QDRANT_READ_ONLY_API_KEY" ]; then
+    echo "ERROR: QDRANT_READ_ONLY_API_KEY environment variable is required"
+    echo "Please set environment variables before running deployment"
+    exit 1
+fi
+
+# Set defaults for environment variables
+MAX_JOBS=${MAX_JOBS:-12}
+
+echo "Starting deployment with MAX_JOBS=$MAX_JOBS"
+echo "Using Qdrant API keys from environment variables"
+
+# update, upgrade and install some essentials
+apt update && apt upgrade -y && apt install -y wget curl git build-essential libcurl4-openssl-dev nginx
+## install torch latest
+pip install --upgrade --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128
+## Process manager
+npm i -g pm2
+# download qdrant vector database
+mkdir qdrant && cd qdrant
+wget https://github.com/qdrant/qdrant/releases/download/v1.2.0/qdrant-x86_64-unknown-linux-gnu.tar.gz
+tar -xzvf qdrant-x86_64-unknown-linux-gnu.tar.gz
+# create qdrant config
+cat > config.yaml << EOF
+log_level: INFO
+storage:
+  storage_path: /data/qdrant
+  snapshots_path: /data/snapshots
+  snapshots_config:
+    snapshots_storage: local
+  temp_path: null
+  on_disk_payload: true
+  update_concurrency: null
+  wal:
+    wal_capacity_mb: 32
+    wal_segments_ahead: 0
+  node_type: "Normal"
+  optimizers:
+    deleted_threshold: 0.2
+    vacuum_min_vector_number: 1000
+    indexing_threshold_kb: 20000
+    flush_interval_sec: 5
+    max_optimization_threads: null
+  hnsw_index:
+    m: 16
+    ef_construct: 100
+    full_scan_threshold_kb: 10000
+    max_indexing_threads: 0
+    on_disk: false
+    payload_m: null
+  shard_transfer_method: null
+  collection:
+    replication_factor: 1
+    write_consistency_factor: 1
+    vectors:
+      on_disk: null
+    quantization: null
+    strict_mode:
+      enabled: false
+      max_query_limit: null
+      max_timeout: null
+service:
+  max_request_size_mb: 32
+  max_workers: 0
+  host: 0.0.0.0
+  http_port: 6333
+  grpc_port: 6334
+  enable_cors: false
+  enable_tls: false
+  verify_https_client_certificate: false
+  api_key: $QDRANT_API_KEY
+  read_only_api_key: $QDRANT_READ_ONLY_API_KEY
+EOF
+
+# Start Qdrant node
+pm2 start --name qdrant './qdrant --config-path config.yaml'
+# back to root folder
+cd ..
+## install vLLM
+git clone https://github.com/vllm-project/vllm.git
+cd vllm
+export MAX_JOBS=$MAX_JOBS
+python use_existing_torch.py
+pip install -r requirements/build.txt
+# build
+pip install --no-build-isolation -e .
+# back to root folder
+cd ..
+
+# Start LLM + Embedding Server Router
+git clone https://gitlab.allelo.eco/alpdeniz/vllm_api
+cd vllm_api
+pip install .
+pm2 start --name ai_proxy 'uvicorn app.main:app --host 0.0.0.0 --port 4444'
+
+# Start the model with vLLM
+pm2 start --name vllm 'vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \
+  --dtype half \
+  --quantization gptq \
+  --gpu-memory-utilization 0.7 \
+  --port 8001'