Files
exAichatbot_agent/exAiChatBot-chatbot2.0-agent/scripts/admin_service.py
T
Macbook 4b86b2a660 Agent 2.0 exdev 서버 배포 스택
- server-dev start/stop/deploy 및 Gitea push 자동 배포
- local-dev 로컬 개발 환경

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-21 22:57:30 +09:00

583 lines
23 KiB
Python

"""
admin_service.py
────────────────────────────────────────────
벡터 DB(Qdrant) 큐레이션 어드민 API + 단일 페이지 웹 UI.
기능:
- 검색(semantic): 질문 임베딩 → 벡터 검색
- 목록(browse): 페이지네이션 조회
- 단건 조회 / 삭제 / 일괄 삭제
- 추가: 질문/답변 텍스트 → 임베딩 → 결정적 ID로 upsert
실시간 /ask 서비스(run_service_qa.py)와 같은 모듈/벡터스토어를 공유하되,
별도 프로세스(컨테이너)로 분리해 운영한다. Qdrant 전용 기능을 사용한다.
"""
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional, List, Dict, Any
from urllib.parse import unquote
import numpy as np
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field
from api_clients import TEIEmbeddingClient
from vector_store import get_vector_store, VECTOR_STORE, make_point_id
app = FastAPI(title="RAG 벡터DB 큐레이션 어드민")
# 별도 origin(다른 포트/도메인)의 프론트엔드가 호출할 수 있도록 CORS 허용.
# 내부망 도구이므로 기본 전체 허용. 필요 시 ADMIN_CORS_ORIGINS="http://host:port,..." 로 제한.
_cors = os.getenv("ADMIN_CORS_ORIGINS", "*")
_origins = ["*"] if _cors.strip() == "*" else [o.strip() for o in _cors.split(",") if o.strip()]
app.add_middleware(
CORSMiddleware,
allow_origins=_origins,
allow_methods=["*"],
allow_headers=["*"],
)
DATA_DIR = Path(os.getenv("DATA_DIR", "/app/data"))
def _now_iso() -> str:
"""등록/수정 시각 기본값 (UTC ISO-8601)"""
return datetime.now(timezone.utc).isoformat()
print("[Admin] 초기화 시작...")
embed_client = TEIEmbeddingClient()
vector_store = get_vector_store()
vector_store.load(str(DATA_DIR))
print(f"[Admin] 벡터 스토어({VECTOR_STORE}) 로드 완료: {vector_store.count()}개")
if VECTOR_STORE != "qdrant":
print("[Admin] ⚠️ 경고: 어드민의 조회/삭제/추가 기능은 Qdrant에서만 동작합니다 "
f"(현재 VECTOR_STORE={VECTOR_STORE}).")
else:
vector_store.ensure_admin_payload_indexes()
# ───────────────────────────────────────────
# 요청/응답 모델
# ───────────────────────────────────────────
class SearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=500)
top_k: int = Field(20, ge=1, le=100)
threshold: Optional[float] = None
category: Optional[str] = None # 정확일치 필터
source: Optional[str] = None # 정확일치 필터
class AddRequest(BaseModel):
q: str = Field(..., min_length=1)
a: str = Field(..., min_length=1)
category: Optional[str] = None
url: Optional[str] = None
source: str = "admin_manual"
source_id: Optional[str] = None
source_created_at: Optional[str] = None
class UpdateRequest(BaseModel):
"""전달된 필드만 수정. q가 바뀌면 재임베딩한다."""
q: Optional[str] = None
a: Optional[str] = None
category: Optional[str] = None
url: Optional[str] = None
source_created_at: Optional[str] = None
class KeywordSearchRequest(BaseModel):
"""DB 전체 대상 문자열 포함(부분일치) 검색"""
keyword: str = Field(..., min_length=1, max_length=200)
field: str = Field("both", pattern="^(both|q|a)$") # 검색 대상 필드
category: Optional[str] = None
source: Optional[str] = None
skip: int = Field(0, ge=0)
limit: int = Field(50, ge=1, le=200)
class DeleteBatchRequest(BaseModel):
ids: List[str] = Field(..., min_items=1)
def _row(item: Dict[str, Any], score: Optional[float] = None) -> Dict[str, Any]:
"""검색/조회 결과를 UI 친화 형태로 변환"""
meta = item.get("meta", {}) or {}
row = {
"id": item.get("id"),
"q": meta.get("q"),
"a": meta.get("a"),
"category": meta.get("category"),
"source": meta.get("source"),
"source_id": meta.get("source_id"),
"url": meta.get("url"),
"source_created_at": meta.get("source_created_at"),
"indexed_at": meta.get("indexed_at"),
"updated_at": meta.get("updated_at"),
}
if score is not None:
row["score"] = round(float(score), 4)
return row
def _require_qdrant():
if VECTOR_STORE != "qdrant":
raise HTTPException(
status_code=400,
detail="이 기능은 Qdrant 벡터스토어에서만 지원됩니다. VECTOR_STORE=qdrant 로 실행하세요.",
)
# ───────────────────────────────────────────
# API
# ───────────────────────────────────────────
@app.get("/api/stats")
def stats():
return {"vector_store": VECTOR_STORE, "count": vector_store.count()}
def _normalize_query_param(value: Optional[str]) -> Optional[str]:
"""쿼리 파라미터 이중 URL 인코딩 복구 (Java RestTemplate + toUriString 조합 대응)."""
if value is None:
return None
v = value.strip()
if not v:
return v
for _ in range(3):
decoded = unquote(v)
if decoded == v:
break
v = decoded
return v
def _filter_dict(category: Optional[str], source: Optional[str]) -> Optional[Dict[str, Any]]:
f: Dict[str, Any] = {}
if category:
cat = (_normalize_query_param(category) or category).strip()
if cat == "미분류":
f["category"] = "__EMPTY__"
else:
f["category"] = cat
if source:
src = (_normalize_query_param(source) or source).strip()
if src:
f["source"] = src
return f or None
@app.post("/api/search")
def search(req: SearchRequest):
vecs = embed_client.embed([req.query], normalize=True, is_query=True)
if not vecs:
raise HTTPException(status_code=502, detail="임베딩 실패")
query_vec = np.array(vecs[0], dtype="float32")
results = vector_store.search(
query_vec,
top_k=req.top_k,
threshold=req.threshold,
filter_dict=_filter_dict(req.category, req.source),
)
return {"count": len(results), "items": [_row(r, r.get("score")) for r in results]}
@app.post("/api/keyword-search")
def keyword_search(req: KeywordSearchRequest):
"""문자열 포함(부분일치) 검색 — 의미 검색과 달리 키워드가 실제 포함된 항목만 반환"""
_require_qdrant()
fields = {"both": ("q", "a"), "q": ("q",), "a": ("a",)}[req.field]
items, total, exhausted = vector_store.keyword_search(
req.keyword,
fields=fields,
filter_dict=_filter_dict(req.category, req.source),
skip=req.skip,
limit=req.limit,
)
return {
"count": len(items),
"total": total,
"exhausted": exhausted,
"items": [_row(it) for it in items],
}
@app.get("/api/categories")
def categories():
"""등록된 category 목록 + 건수 (어드민 분류 필터/트리용)"""
_require_qdrant()
counts = vector_store.distinct_payload_values("category")
items = [{"category": k, "count": v} for k, v in sorted(counts.items())]
return {"count": len(items), "items": items}
@app.get("/api/points")
def list_points(
limit: int = 50,
offset: Optional[str] = None,
page: Optional[int] = None,
size: Optional[int] = None,
category: Optional[str] = None,
source: Optional[str] = None,
):
_require_qdrant()
fd = _filter_dict(category, source)
if page is not None:
pg = max(0, page)
sz = max(1, min(size or 20, 200))
items, total = vector_store.list_points_page(pg, sz, fd)
return {
"page": pg,
"size": sz,
"total": total,
"items": [_row(it) for it in items],
}
items, next_offset = vector_store.list_points(
limit=limit, offset=offset, filter_dict=fd
)
return {
"count": len(items),
"items": [_row(it) for it in items],
"next_offset": next_offset,
}
@app.get("/api/points/{point_id}")
def get_point(point_id: str):
_require_qdrant()
item = vector_store.get_by_id(point_id)
if not item:
raise HTTPException(status_code=404, detail="해당 ID의 항목이 없습니다.")
return _row(item)
@app.put("/api/points/{point_id}")
def update_point(point_id: str, req: UpdateRequest):
"""
기존 항목 수정. 전달된 필드만 갱신한다.
- q 기반 결정적 ID → q 변경 시 새 ID로 upsert 후 옛 ID 삭제
- 추가/업로드와 동일: 더 최신 source_created_at이 이미 있으면 스킵
"""
_require_qdrant()
existing = vector_store.get_by_id(point_id)
if not existing:
raise HTTPException(status_code=404, detail="해당 ID의 항목이 없습니다.")
meta: Dict[str, Any] = dict(existing.get("meta") or {})
if req.q is not None:
meta["q"] = req.q.strip()
if req.a is not None:
meta["a"] = req.a
if req.category is not None:
meta["category"] = req.category
if req.url is not None:
meta["url"] = req.url
if req.source_created_at is not None:
meta["source_created_at"] = req.source_created_at.strip() or None
if not meta.get("q") or not meta.get("a"):
raise HTTPException(status_code=400, detail="q와 a는 비울 수 없습니다.")
if not meta.get("source_created_at") or not str(meta.get("source_created_at")).strip():
raise HTTPException(status_code=400, detail="source_created_at은 필수입니다.")
meta["source_created_at"] = str(meta["source_created_at"]).strip()
# 질문 재임베딩 (q가 안 바뀌었어도 일관성을 위해 항상 재임베딩)
vecs = embed_client.embed([meta["q"]], normalize=True, is_query=False)
if not vecs:
raise HTTPException(status_code=502, detail="임베딩 실패")
new_id = make_point_id(meta)
vectors = np.array([vecs[0]], dtype="float32")
if hasattr(vector_store, "upsert_vectors"):
stats = vector_store.upsert_vectors(vectors, [meta], skip_if_older=True)
if stats.get("skipped"):
return {
"skipped": True,
"reason": stats.get("last_skip_reason") or "older_source_created_at",
"id": stats.get("last_id") or new_id,
}
else:
meta.setdefault("indexed_at", _now_iso())
meta["updated_at"] = _now_iso()
vector_store.add_vectors(vectors, [meta], skip_if_older=True)
# ID가 바뀐 경우(질문 변경) 옛 항목 제거
if new_id != point_id:
vector_store.delete_by_id(point_id)
# 레거시(source_id 기반) ID로 남아 있는 동일 q 항목 정리
if hasattr(vector_store, "find_by_exact_q"):
legacy = vector_store.find_by_exact_q(meta["q"])
if legacy and str(legacy.get("id")) not in (str(new_id), str(point_id)):
vector_store.delete_by_id(str(legacy["id"]))
return {"updated": new_id, "moved": new_id != point_id, "skipped": False}
@app.delete("/api/points/{point_id}")
def delete_point(point_id: str):
_require_qdrant()
if not vector_store.get_by_id(point_id):
raise HTTPException(status_code=404, detail="해당 ID의 항목이 없습니다.")
vector_store.delete_by_id(point_id)
return {"deleted": point_id}
@app.post("/api/points/delete-batch")
def delete_batch(req: DeleteBatchRequest):
_require_qdrant()
vector_store.delete_by_ids(req.ids)
return {"deleted": len(req.ids)}
@app.post("/api/points")
def add_point(req: AddRequest):
_require_qdrant()
if not req.source_created_at or not req.source_created_at.strip():
raise HTTPException(status_code=400, detail="source_created_at은 필수입니다.")
# 문서 임베딩(is_query=False) → ingest와 동일 방식
vecs = embed_client.embed([req.q], normalize=True, is_query=False)
if not vecs:
raise HTTPException(status_code=502, detail="임베딩 실패")
source_id = req.source_id or f"manual_{os.urandom(6).hex()}"
now = _now_iso()
meta: Dict[str, Any] = {
"q": req.q.strip(),
"a": req.a,
"category": req.category,
"source": req.source,
"source_id": source_id,
"url": req.url,
"source_created_at": req.source_created_at.strip(),
"indexed_at": now,
"updated_at": now,
}
point_id = make_point_id(meta)
vectors = np.array([vecs[0]], dtype="float32")
if hasattr(vector_store, "upsert_vectors"):
stats = vector_store.upsert_vectors(vectors, [meta], skip_if_older=True)
if stats.get("skipped"):
return {
"skipped": True,
"reason": stats.get("last_skip_reason") or "older_source_created_at",
"id": stats.get("last_id") or point_id,
}
action = stats.get("last_action") or "inserted"
return {
"created": stats.get("last_id") or point_id,
"source_id": source_id,
"action": action,
"skipped": False,
}
vector_store.add_vectors(vectors, [meta], skip_if_older=True)
return {"created": point_id, "source_id": source_id, "skipped": False}
# ───────────────────────────────────────────
# 웹 UI (빌드 불필요 단일 페이지)
# ───────────────────────────────────────────
@app.get("/", response_class=HTMLResponse)
def index():
return HTML_PAGE
HTML_PAGE = """<!doctype html>
<html lang="ko">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>벡터DB 큐레이션 어드민</title>
<style>
:root { --bd:#e2e8f0; --pri:#2563eb; --bg:#f8fafc; --txt:#0f172a; --muted:#64748b; --danger:#dc2626; }
* { box-sizing:border-box; }
body { margin:0; font-family:'Segoe UI',-apple-system,system-ui,sans-serif; background:var(--bg); color:var(--txt); }
header { background:#fff; border-bottom:1px solid var(--bd); padding:16px 24px; display:flex; align-items:center; gap:16px; position:sticky; top:0; z-index:10; }
header h1 { font-size:18px; margin:0; }
.stat { margin-left:auto; color:var(--muted); font-size:14px; }
main { max-width:1100px; margin:0 auto; padding:24px; }
.tabs { display:flex; gap:8px; margin-bottom:16px; }
.tab { padding:8px 16px; border:1px solid var(--bd); border-radius:8px; background:#fff; cursor:pointer; font-size:14px; }
.tab.active { background:var(--pri); color:#fff; border-color:var(--pri); }
.panel { display:none; background:#fff; border:1px solid var(--bd); border-radius:12px; padding:20px; }
.panel.active { display:block; }
.row { display:flex; gap:8px; align-items:center; flex-wrap:wrap; }
input, textarea, select { width:100%; padding:9px 12px; border:1px solid var(--bd); border-radius:8px; font-size:14px; font-family:inherit; }
textarea { min-height:90px; resize:vertical; }
label { display:block; font-size:13px; color:var(--muted); margin:10px 0 4px; }
button { padding:9px 16px; border:none; border-radius:8px; background:var(--pri); color:#fff; cursor:pointer; font-size:14px; }
button.secondary { background:#fff; color:var(--txt); border:1px solid var(--bd); }
button.danger { background:var(--danger); }
button:disabled { opacity:.5; cursor:not-allowed; }
.card { border:1px solid var(--bd); border-radius:10px; padding:14px; margin-top:12px; }
.card .meta { font-size:12px; color:var(--muted); margin-bottom:6px; display:flex; gap:10px; flex-wrap:wrap; }
.card .q { font-weight:600; margin-bottom:6px; }
.card .a { font-size:14px; color:#334155; white-space:pre-wrap; }
.badge { background:#eff6ff; color:var(--pri); border-radius:6px; padding:1px 8px; font-size:12px; }
.toolbar { display:flex; gap:8px; margin-top:10px; }
.empty { color:var(--muted); text-align:center; padding:30px; }
.toast { position:fixed; bottom:20px; left:50%; transform:translateX(-50%); background:#0f172a; color:#fff; padding:10px 18px; border-radius:8px; opacity:0; transition:.2s; font-size:14px; }
.toast.show { opacity:1; }
code { background:#f1f5f9; padding:1px 6px; border-radius:4px; font-size:12px; }
</style>
</head>
<body>
<header>
<h1>🗂️ 벡터DB 큐레이션 어드민</h1>
<div class="stat" id="stat">로딩...</div>
</header>
<main>
<div class="tabs">
<div class="tab active" data-tab="search">의미검색</div>
<div class="tab" data-tab="keyword">키워드검색</div>
<div class="tab" data-tab="browse">목록</div>
<div class="tab" data-tab="add">추가</div>
</div>
<!-- 의미 검색 -->
<section class="panel active" id="panel-search">
<div class="row">
<input id="q" placeholder="질문을 입력하세요 (뜻이 비슷한 항목 검색 = 챗봇과 동일)" onkeydown="if(event.key==='Enter')doSearch()"/>
<button onclick="doSearch()">검색</button>
</div>
<div id="search-results"></div>
</section>
<!-- 키워드(문자열 포함) 검색 -->
<section class="panel" id="panel-keyword">
<div class="row">
<input id="kw" placeholder="포함된 문자열 검색 (예: 1588-2504, 부가통행료)" onkeydown="if(event.key==='Enter')doKeyword()"/>
<select id="kw-field" style="max-width:140px">
<option value="both">질문+답변</option>
<option value="q">질문만</option>
<option value="a">답변만</option>
</select>
<button onclick="doKeyword()">검색</button>
</div>
<div id="kw-results"></div>
</section>
<!-- 목록 -->
<section class="panel" id="panel-browse">
<div class="row">
<button class="secondary" onclick="loadList(true)">처음부터</button>
<button class="secondary" id="more-btn" onclick="loadList(false)">더 보기</button>
</div>
<div id="list-results"></div>
</section>
<!-- 추가 -->
<section class="panel" id="panel-add">
<label>질문 (q) *</label>
<input id="add-q" placeholder="예: 하이패스 단말기는 어디서 구입하나요?"/>
<label>답변 (a) *</label>
<textarea id="add-a" placeholder="답변 내용"></textarea>
<div class="row">
<div style="flex:1"><label>분류 (category)</label><input id="add-cat" placeholder="예: 하이패스 이용"/></div>
<div style="flex:1"><label>URL</label><input id="add-url" placeholder="https://..."/></div>
</div>
<div class="toolbar"><button onclick="doAdd()">벡터DB에 추가</button></div>
<p style="color:var(--muted);font-size:13px">추가 시 질문이 임베딩되어 검색 대상이 됩니다. 같은 항목을 다시 추가하면 덮어쓰기됩니다.</p>
</section>
</main>
<div class="toast" id="toast"></div>
<script>
const $ = (s) => document.querySelector(s);
let listOffset = null;
function toast(msg){ const t=$("#toast"); t.textContent=msg; t.classList.add("show"); setTimeout(()=>t.classList.remove("show"),2200); }
function esc(s){ return (s??"").toString().replace(/[&<>]/g, c=>({'&':'&amp;','<':'&lt;','>':'&gt;'}[c])); }
document.querySelectorAll(".tab").forEach(t=>t.onclick=()=>{
document.querySelectorAll(".tab").forEach(x=>x.classList.remove("active"));
document.querySelectorAll(".panel").forEach(x=>x.classList.remove("active"));
t.classList.add("active");
$("#panel-"+t.dataset.tab).classList.add("active");
if(t.dataset.tab==="browse") loadList(true);
});
async function refreshStat(){
try{ const r=await fetch("api/stats"); const d=await r.json(); $("#stat").textContent=`벡터 ${d.count.toLocaleString()}개 · ${d.vector_store}`; }
catch(e){ $("#stat").textContent="상태 조회 실패"; }
}
function cardHtml(it, showScore){
const score = showScore && it.score!=null ? `<span class="badge">score ${it.score}</span>` : "";
return `<div class="card" data-id="${esc(it.id)}">
<div class="meta">${score}<span>출처: ${esc(it.source)||'-'}</span><span>분류: ${esc(it.category)||'-'}</span><span>id: <code>${esc(it.id)}</code></span></div>
<div class="q">Q. ${esc(it.q)}</div>
<div class="a">A. ${esc(it.a)}</div>
${it.url?`<div class="meta"><a href="${esc(it.url)}" target="_blank">${esc(it.url)}</a></div>`:""}
<div class="toolbar"><button class="danger" onclick="del('${esc(it.id)}')">삭제</button></div>
</div>`;
}
async function doSearch(){
const q=$("#q").value.trim(); if(!q) return;
$("#search-results").innerHTML="<div class='empty'>검색 중...</div>";
try{
const r=await fetch("api/search",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({query:q,top_k:20})});
const d=await r.json();
$("#search-results").innerHTML = d.items.length ? d.items.map(it=>cardHtml(it,true)).join("") : "<div class='empty'>결과가 없습니다.</div>";
}catch(e){ $("#search-results").innerHTML="<div class='empty'>검색 실패</div>"; }
}
async function doKeyword(){
const kw=$("#kw").value.trim(); if(!kw) return;
$("#kw-results").innerHTML="<div class='empty'>검색 중...</div>";
try{
const r=await fetch("api/keyword-search",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({keyword:kw,field:$("#kw-field").value,limit:100})});
const d=await r.json();
const note=`<div class="meta" style="margin-bottom:8px">스캔 ${d.scanned}건 · 매칭 ${d.count}건${d.exhausted?'':' (상위 일부만 표시)'}</div>`;
$("#kw-results").innerHTML = note + (d.items.length ? d.items.map(it=>cardHtml(it,false)).join("") : "<div class='empty'>포함된 항목이 없습니다.</div>");
}catch(e){ $("#kw-results").innerHTML="<div class='empty'>검색 실패</div>"; }
}
async function loadList(reset){
if(reset){ listOffset=null; $("#list-results").innerHTML=""; }
try{
const url = "api/points?limit=20" + (listOffset?("&offset="+encodeURIComponent(listOffset)):"");
const r=await fetch(url); const d=await r.json();
if(reset && !d.items.length){ $("#list-results").innerHTML="<div class='empty'>등록된 항목이 없습니다.</div>"; }
else { $("#list-results").insertAdjacentHTML("beforeend", d.items.map(it=>cardHtml(it,false)).join("")); }
listOffset = d.next_offset;
$("#more-btn").disabled = !listOffset;
}catch(e){ toast("목록 조회 실패"); }
}
async function del(id){
if(!confirm("이 항목을 벡터DB에서 삭제할까요?")) return;
try{
const r=await fetch("api/points/"+encodeURIComponent(id),{method:"DELETE"});
if(!r.ok) throw 0;
document.querySelector(`.card[data-id="${CSS.escape(id)}"]`)?.remove();
toast("삭제되었습니다."); refreshStat();
}catch(e){ toast("삭제 실패"); }
}
async function doAdd(){
const q=$("#add-q").value.trim(), a=$("#add-a").value.trim();
if(!q||!a){ toast("질문과 답변은 필수입니다."); return; }
try{
const r=await fetch("api/points",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({q,a,category:$("#add-cat").value.trim()||null,url:$("#add-url").value.trim()||null})});
if(!r.ok) throw 0;
$("#add-q").value=""; $("#add-a").value=""; $("#add-cat").value=""; $("#add-url").value="";
toast("추가되었습니다."); refreshStat();
}catch(e){ toast("추가 실패"); }
}
refreshStat();
</script>
</body>
</html>"""