Agent 2.0 exdev 서버 배포 스택
- server-dev start/stop/deploy 및 Gitea push 자동 배포 - local-dev 로컬 개발 환경 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
=========== embedding ===========
|
||||
|
||||
docker run -d --gpus all \
|
||||
--name tei-embedding \
|
||||
-p 16001:80 \
|
||||
-v /DATA/exlink/models:/data \
|
||||
--pull never \
|
||||
--restart always \
|
||||
ghcr.io/huggingface/text-embeddings-inference:hopper-1.8 \
|
||||
--model-id /data/qwen3-embedding-8b \
|
||||
--port 80 \
|
||||
--max-batch-tokens 40960
|
||||
|
||||
|
||||
curl 127.0.0.1:16001/embed \
|
||||
-X POST \
|
||||
-d '{"inputs":"안녕하세요, 임베딩 테스트 중입니다."}' \
|
||||
-H 'Content-Type: application/json'
|
||||
=========== embedding ===========
|
||||
|
||||
=========== reranker ===========
|
||||
|
||||
docker rm -f vllm-reranker
|
||||
|
||||
docker rm -f vllm-reranker
|
||||
|
||||
docker run -d --gpus all \
|
||||
--name vllm-reranker \
|
||||
-p 16002:8000 \
|
||||
-v /DATA/exlink/models:/data \
|
||||
--ipc=host \
|
||||
--restart always \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-e TRANSFORMERS_OFFLINE=1 \
|
||||
vllm/vllm-openai:latest \
|
||||
python3 /data/reranker_server.py
|
||||
|
||||
curl http://127.0.0.1:16002/v1/rerank \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "/data/qwen3-reranker-4b",
|
||||
"query": "사과는 어떤 과일인가요?",
|
||||
"documents": [
|
||||
"사과는 빨간색 과일이며 비타민이 풍부합니다.",
|
||||
"서울은 대한민국의 수도입니다."
|
||||
]
|
||||
}'
|
||||
=========== reranker ===========
|
||||
|
||||
|
||||
=========== LLM ===========
|
||||
|
||||
docker run -d --gpus all \
|
||||
--name vllm-server \
|
||||
--memory 32g \
|
||||
-p 16000:8000 \
|
||||
-v /DATA/exlink/models:/data \
|
||||
--ipc=host \ --restart always \
|
||||
--network bridge \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-e HF_HOME=/data/cache \
|
||||
-e TRANSFORMERS_OFFLINE=1 \
|
||||
--entrypoint python3 \
|
||||
vllm/vllm-openai:latest \
|
||||
-m vllm.entrypoints.openai.api_server \
|
||||
--model /data/Qwen3.5-27B-FP8 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--gpu-memory-utilization 0.65 \
|
||||
--max-model-len 5120 \
|
||||
--max-num-seqs 8 \
|
||||
--enable-chunked-prefill \
|
||||
--max-num-batched-tokens 4096 \
|
||||
|
||||
=========== LLM ===========
|
||||
|
||||
|
||||
Reference in New Issue
Block a user