Macbook
4b86b2a660
- server-dev start/stop/deploy 및 Gitea push 자동 배포 - local-dev 로컬 개발 환경 Co-authored-by: Cursor <cursoragent@cursor.com>
79 lines
1.8 KiB
Plaintext
79 lines
1.8 KiB
Plaintext
=========== embedding ===========
|
|
|
|
docker run -d --gpus all \
|
|
--name tei-embedding \
|
|
-p 16001:80 \
|
|
-v /DATA/exlink/models:/data \
|
|
--pull never \
|
|
--restart always \
|
|
ghcr.io/huggingface/text-embeddings-inference:hopper-1.8 \
|
|
--model-id /data/qwen3-embedding-8b \
|
|
--port 80 \
|
|
--max-batch-tokens 40960
|
|
|
|
|
|
curl 127.0.0.1:16001/embed \
|
|
-X POST \
|
|
-d '{"inputs":"안녕하세요, 임베딩 테스트 중입니다."}' \
|
|
-H 'Content-Type: application/json'
|
|
=========== embedding ===========
|
|
|
|
=========== reranker ===========
|
|
|
|
docker rm -f vllm-reranker
|
|
|
|
docker rm -f vllm-reranker
|
|
|
|
docker run -d --gpus all \
|
|
--name vllm-reranker \
|
|
-p 16002:8000 \
|
|
-v /DATA/exlink/models:/data \
|
|
--ipc=host \
|
|
--restart always \
|
|
-e HF_HUB_OFFLINE=1 \
|
|
-e TRANSFORMERS_OFFLINE=1 \
|
|
vllm/vllm-openai:latest \
|
|
python3 /data/reranker_server.py
|
|
|
|
curl http://127.0.0.1:16002/v1/rerank \
|
|
-X POST \
|
|
-H "Content-Type: application/json" \
|
|
-d '{
|
|
"model": "/data/qwen3-reranker-4b",
|
|
"query": "사과는 어떤 과일인가요?",
|
|
"documents": [
|
|
"사과는 빨간색 과일이며 비타민이 풍부합니다.",
|
|
"서울은 대한민국의 수도입니다."
|
|
]
|
|
}'
|
|
=========== reranker ===========
|
|
|
|
|
|
=========== LLM ===========
|
|
|
|
docker run -d --gpus all \
|
|
--name vllm-server \
|
|
--memory 32g \
|
|
-p 16000:8000 \
|
|
-v /DATA/exlink/models:/data \
|
|
--ipc=host \ --restart always \
|
|
--network bridge \
|
|
-e HF_HUB_OFFLINE=1 \
|
|
-e HF_HOME=/data/cache \
|
|
-e TRANSFORMERS_OFFLINE=1 \
|
|
--entrypoint python3 \
|
|
vllm/vllm-openai:latest \
|
|
-m vllm.entrypoints.openai.api_server \
|
|
--model /data/Qwen3.5-27B-FP8 \
|
|
--host 0.0.0.0 \
|
|
--port 8000 \
|
|
--gpu-memory-utilization 0.65 \
|
|
--max-model-len 5120 \
|
|
--max-num-seqs 8 \
|
|
--enable-chunked-prefill \
|
|
--max-num-batched-tokens 4096 \
|
|
|
|
=========== LLM ===========
|
|
|
|
|