llm 동작 수정

This commit is contained in:
Macbook
2026-06-17 15:37:23 +09:00
parent 301f45301f
commit edac7540cd
14 changed files with 901 additions and 63 deletions
+45 -1
View File
@@ -1738,7 +1738,51 @@ export interface StrategyEvaluationAiVerifyResponse {
ta4jDiagnostics?: Record<string, unknown> | null;
}
/** 전략 평가 — LLM 기반 Rule·시그널 일치 검증 */
export type StrategyEvaluationAiVerifyJobStatus =
| 'QUEUED'
| 'RUNNING'
| 'COMPLETED'
| 'FAILED';
export interface StrategyEvaluationAiVerifyJobStartResponse {
jobId: string;
status: StrategyEvaluationAiVerifyJobStatus;
}
export interface StrategyEvaluationAiVerifyJobStatusResponse {
jobId: string;
status: StrategyEvaluationAiVerifyJobStatus;
error?: string | null;
result?: StrategyEvaluationAiVerifyResponse | null;
createdAtMs: number;
completedAtMs?: number | null;
}
/** 전략 평가 — AI 검증 비동기 job 시작 (즉시 jobId 반환) */
export async function startStrategyEvaluationAiVerifyJob(
context: Record<string, unknown>,
): Promise<StrategyEvaluationAiVerifyJobStartResponse> {
return requestOrThrow<StrategyEvaluationAiVerifyJobStartResponse>(
'/strategy/evaluation/ai-verify/jobs',
{
method: 'POST',
cache: 'no-store',
body: JSON.stringify({ context }),
},
);
}
/** AI 검증 job 상태 조회 */
export async function fetchStrategyEvaluationAiVerifyJob(
jobId: string,
): Promise<StrategyEvaluationAiVerifyJobStatusResponse> {
return requestOrThrow<StrategyEvaluationAiVerifyJobStatusResponse>(
`/strategy/evaluation/ai-verify/jobs/${encodeURIComponent(jobId)}`,
{ cache: 'no-store' },
);
}
/** 전략 평가 — LLM 기반 Rule·시그널 일치 검증 (레거시 동기) */
export async function fetchStrategyEvaluationAiVerify(
context: Record<string, unknown>,
): Promise<StrategyEvaluationAiVerifyResponse> {
+117 -34
View File
@@ -2,7 +2,11 @@
* 전략 평가 AI 검증 — LLM 전달용 컨텍스트 구성 및 API 호출
*/
import type { BacktestSignal, StrategyDto, StrategyEvaluationAiVerifyResponse } from './backendApi';
import { fetchStrategyEvaluationAiVerify } from './backendApi';
import {
fetchStrategyEvaluationAiVerifyJob,
startStrategyEvaluationAiVerifyJob,
type StrategyEvaluationAiVerifyJobStatusResponse,
} from './backendApi';
import type { OHLCVBar } from '../types';
import type { VirtualIndicatorSnapshot } from '../hooks/useVirtualIndicatorSnapshots';
import type { BarSignalHighlight } from './strategyEvaluationBarSignals';
@@ -12,6 +16,13 @@ import { formatVirtualConditionListLabel } from './virtualStrategyConditions';
import { buildConditionMetrics } from './virtualSignalMetrics';
import { repairUtf8Mojibake } from './textEncoding';
import { normalizeEpochSecOptional } from './backtestUiUtils';
import {
buildLlmVerifyPayload,
formatLlmConditionLines,
indexStrategyConditions,
enrichConditionRow,
type LlmConditionEvaluationRow,
} from './strategyEvaluationLlmContext';
export interface StrategyEvaluationAiVerifyContext {
meta: {
@@ -285,8 +296,71 @@ function signalMatchesBar(
export async function requestStrategyEvaluationAiVerify(
context: StrategyEvaluationAiVerifyContext,
opts?: {
signal?: AbortSignal;
onJobStarted?: (jobId: string) => void;
pollIntervalMs?: number;
maxWaitMs?: number;
},
): Promise<StrategyEvaluationAiVerifyResponse> {
return fetchStrategyEvaluationAiVerify(context as unknown as Record<string, unknown>);
const payload = buildLlmVerifyPayload(context);
const { jobId } = await startStrategyEvaluationAiVerifyJob(
payload as unknown as Record<string, unknown>,
);
opts?.onJobStarted?.(jobId);
const job = await pollStrategyEvaluationAiVerifyJob(jobId, {
signal: opts?.signal,
pollIntervalMs: opts?.pollIntervalMs ?? 2_500,
maxWaitMs: opts?.maxWaitMs ?? 620_000,
});
if (!job.result) {
throw new Error(job.error ?? 'AI 검증 결과가 없습니다.');
}
return job.result;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new DOMException('Aborted', 'AbortError'));
return;
}
const timer = window.setTimeout(resolve, ms);
const onAbort = () => {
window.clearTimeout(timer);
reject(new DOMException('Aborted', 'AbortError'));
};
signal?.addEventListener('abort', onAbort, { once: true });
});
}
/** 백그라운드 폴링 — job 완료까지 주기적으로 상태 조회 */
export async function pollStrategyEvaluationAiVerifyJob(
jobId: string,
opts?: {
signal?: AbortSignal;
pollIntervalMs?: number;
maxWaitMs?: number;
onStatus?: (job: StrategyEvaluationAiVerifyJobStatusResponse) => void;
},
): Promise<StrategyEvaluationAiVerifyJobStatusResponse> {
const interval = opts?.pollIntervalMs ?? 2_500;
const maxWait = opts?.maxWaitMs ?? 620_000;
const started = Date.now();
while (Date.now() - started < maxWait) {
if (opts?.signal?.aborted) {
throw new DOMException('Aborted', 'AbortError');
}
const job = await fetchStrategyEvaluationAiVerifyJob(jobId);
opts?.onStatus?.(job);
if (job.status === 'COMPLETED') return job;
if (job.status === 'FAILED') {
throw new Error(job.error ?? 'AI 검증 실패');
}
await sleep(interval, opts?.signal);
}
throw new Error('AI 검증 시간 초과. 잠시 후 다시 시도해 주세요.');
}
/** UI 요약 — 컨텍스트 미리보기 */
@@ -324,16 +398,28 @@ export interface AiVerifyFixPointsBundle {
cursorPrompt: string;
}
function formatConditionLines(
conditions: StrategyEvaluationAiVerifyContext['evaluation']['buyConditions'],
): string {
if (conditions.length === 0) return '(조건 없음)';
return conditions.map(c => {
const status = c.satisfied === true ? '충족' : c.satisfied === false ? '미충족' : '평가불가';
const cur = c.currentValue != null ? String(c.currentValue) : '-';
const target = c.thresholdLabel ?? (c.targetValue != null ? String(c.targetValue) : '-');
return `- [${c.id}] ${c.label} (${c.timeframe}/${c.indicatorType}): ${status} · 현재=${cur} · 기준=${target}`;
}).join('\n');
function llmConditionRows(ctx: StrategyEvaluationAiVerifyContext): {
buy: LlmConditionEvaluationRow[];
sell: LlmConditionEvaluationRow[];
} {
const dslById = indexStrategyConditions(ctx.strategy.buyCondition, ctx.strategy.sellCondition);
return {
buy: ctx.evaluation.buyConditions.map(r => enrichConditionRow(r, dslById)),
sell: ctx.evaluation.sellConditions.map(r => enrichConditionRow(r, dslById)),
};
}
function formatConditionLines(ctx: StrategyEvaluationAiVerifyContext): {
buy: string;
sell: string;
all: string;
} {
const { buy, sell } = llmConditionRows(ctx);
return {
buy: formatLlmConditionLines(buy),
sell: formatLlmConditionLines(sell),
all: formatLlmConditionLines([...buy, ...sell]),
};
}
function formatDslSummary(condition: unknown): string {
@@ -397,7 +483,7 @@ function extractVerdictHint(analysis: string | null | undefined): string | null
}
function pushBuySignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const { overallEntryMet, buyConditions } = ctx.evaluation;
const { overallEntryMet } = ctx.evaluation;
const buyHighlight = ctx.signals.selectedBarHighlight.buy;
if (overallEntryMet !== true || buyHighlight) return;
@@ -409,7 +495,7 @@ function pushBuySignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points: A
'',
'【매수 조건 평가】',
`overallEntryMet: true (전체 매수 Rule 충족)`,
formatConditionLines(buyConditions),
formatConditionLines(ctx).buy,
'',
'【선택 봉 차트 시그널】',
`BUY: 없음 · SELL: ${ctx.signals.selectedBarHighlight.sell ? '있음' : '없음'}`,
@@ -438,7 +524,7 @@ function pushBuySignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points: A
}
function pushSellSignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const { overallExitMet, sellConditions } = ctx.evaluation;
const { overallExitMet } = ctx.evaluation;
const sellHighlight = ctx.signals.selectedBarHighlight.sell;
if (overallExitMet !== true || sellHighlight) return;
@@ -450,7 +536,7 @@ function pushSellSignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points:
'',
'【매도 조건 평가】',
`overallExitMet: true (전체 매도 Rule 충족)`,
formatConditionLines(sellConditions),
formatConditionLines(ctx).sell,
'',
'【선택 봉 차트 시그널】',
`BUY: ${ctx.signals.selectedBarHighlight.buy ? '있음' : '없음'} · SELL: 없음`,
@@ -476,7 +562,7 @@ function pushSellSignalMismatch(ctx: StrategyEvaluationAiVerifyContext, points:
}
function pushUnexpectedBuySignal(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const { overallEntryMet, buyConditions } = ctx.evaluation;
const { overallEntryMet } = ctx.evaluation;
if (overallEntryMet !== false || !ctx.signals.selectedBarHighlight.buy) return;
points.push({
@@ -487,7 +573,7 @@ function pushUnexpectedBuySignal(ctx: StrategyEvaluationAiVerifyContext, points:
'',
'【매수 조건 평가】',
'overallEntryMet: false',
formatConditionLines(buyConditions),
formatConditionLines(ctx).buy,
'',
'【선택 봉 시그널】',
ctx.signals.selectedBarSignals.filter(s => s.type === 'BUY').map(s =>
@@ -509,7 +595,7 @@ function pushUnexpectedBuySignal(ctx: StrategyEvaluationAiVerifyContext, points:
}
function pushUnexpectedSellSignal(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const { overallExitMet, sellConditions } = ctx.evaluation;
const { overallExitMet } = ctx.evaluation;
if (overallExitMet !== false || !ctx.signals.selectedBarHighlight.sell) return;
points.push({
@@ -520,7 +606,7 @@ function pushUnexpectedSellSignal(ctx: StrategyEvaluationAiVerifyContext, points
'',
'【매도 조건 평가】',
'overallExitMet: false',
formatConditionLines(sellConditions),
formatConditionLines(ctx).sell,
].join('\n'),
settingsAndExpected: [
'전략 매도 DSL:',
@@ -536,10 +622,8 @@ function pushUnexpectedSellSignal(ctx: StrategyEvaluationAiVerifyContext, points
}
function pushFailedConditions(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const failed = [
...ctx.evaluation.buyConditions.filter(c => c.satisfied === false),
...ctx.evaluation.sellConditions.filter(c => c.satisfied === false),
];
const { buy, sell } = llmConditionRows(ctx);
const failed = [...buy, ...sell].filter(c => c.satisfied === false);
if (failed.length === 0) return;
const entryMet = ctx.evaluation.overallEntryMet;
@@ -555,7 +639,7 @@ function pushFailedConditions(ctx: StrategyEvaluationAiVerifyContext, points: Ai
`overallEntryMet: ${entryMet} · overallExitMet: ${exitMet}`,
'',
'【미충족 개별 조건】',
formatConditionLines(failed),
formatLlmConditionLines(failed),
].join('\n'),
settingsAndExpected: [
'전략 DSL (매수/매도):',
@@ -571,10 +655,8 @@ function pushFailedConditions(ctx: StrategyEvaluationAiVerifyContext, points: Ai
}
function pushUnevaluatedConditions(ctx: StrategyEvaluationAiVerifyContext, points: AiVerifyFixPoint[]) {
const unknown = [
...ctx.evaluation.buyConditions.filter(c => c.satisfied == null),
...ctx.evaluation.sellConditions.filter(c => c.satisfied == null),
];
const { buy, sell } = llmConditionRows(ctx);
const unknown = [...buy, ...sell].filter(c => c.satisfied == null);
if (unknown.length === 0) return;
points.push({
@@ -584,7 +666,7 @@ function pushUnevaluatedConditions(ctx: StrategyEvaluationAiVerifyContext, point
chartSituationSummary(ctx),
'',
'【평가 불가 조건】',
formatConditionLines(unknown),
formatLlmConditionLines(unknown),
].join('\n'),
settingsAndExpected: [
'지표 파라미터:',
@@ -657,19 +739,20 @@ export function buildAiVerifyFixPoints(
const issueFromHints = ctx.deterministicHints.signalMatchesOverallRule
!== '전체 Rule과 선택 봉 시그널 표시가 대체로 일치';
if (points.length === 0 && issueFromAnalysis) {
if (points.length === 0 && issueFromAnalysis && issueFromHints) {
const condLines = formatConditionLines(ctx);
points.push({
id: 'llm-verdict-only',
title: 'AI가 로직 문제로 판단',
situation: [
chartSituationSummary(ctx),
'',
'【조건 요약】',
'【조건 요약 — DSL·Ta4j 기준】',
`매수 overallEntryMet: ${ctx.evaluation.overallEntryMet}`,
formatConditionLines(ctx.evaluation.buyConditions),
condLines.buy,
'',
`매도 overallExitMet: ${ctx.evaluation.overallExitMet}`,
formatConditionLines(ctx.evaluation.sellConditions),
condLines.sell,
'',
`시그널 힌트: ${ctx.deterministicHints.signalMatchesOverallRule}`,
].join('\n'),
@@ -0,0 +1,235 @@
/**
* LLM AI 검증 — 오해 소지 UI 라벨 대신 실제 DSL·Ta4j 평가 의미를 전달
*/
import type { ConditionDSL, LogicNode } from './strategyTypes';
import { CONDITION_LABEL } from './strategyTypes';
import { asLogicNode } from './strategyHydrate';
import { normalizeStartCandleType } from './strategyStartNodes';
import { formatIndicatorValue } from './virtualStrategyConditions';
import { isThresholdSymbol } from './thresholdSymbols';
import type { StrategyEvaluationAiVerifyContext } from './strategyEvaluationAiVerify';
const HL_ROLE_KO: Record<string, string> = {
HL_OVER: '과열선',
HL_MID: '중앙선',
HL_UNDER: '침체선',
};
/** Ta4j Rule.isSatisfied 기준 — conditionType별 의미 */
const EVALUATION_SEMANTICS: Record<string, string> = {
GT: '현재봉 left > right',
LT: '현재봉 left < right',
GTE: '현재봉 left ≥ right',
LTE: '현재봉 left ≤ right',
EQ: '현재봉 left = right',
NEQ: '현재봉 left ≠ right',
CROSS_UP:
'직전봉 left≤right 이고 현재봉 left>right (교차 이벤트). 현재값이 임계값보다 커도 이미 돌파된 상태면 미충족.',
CROSS_DOWN:
'직전봉 left≥right 이고 현재봉 left<right (교차 이벤트). 현재값이 임계값보다 작지 않으면 미충족.',
SLOPE_UP: '최근 N봉 기울기 상승',
SLOPE_DOWN: '최근 N봉 기울기 하락',
};
export interface LlmConditionEvaluationRow {
id: string;
side: 'buy' | 'sell';
timeframe: string;
indicatorType: string;
conditionType: string;
satisfied: boolean | null;
currentValue: number | null;
/** leaf condition DSL (strategy JSON 그대로) */
dslCondition: ConditionDSL;
/** Ta4j 평가식 요약 — GT/LT와 혼동되지 않도록 conditionType 명시 */
dslExpression: string;
/** Ta4j isSatisfied 판정 기준 설명 */
evaluationSemantics: string;
/** right operand 해석 (HL_UNDER → 20 등) */
rightOperand: string;
}
export interface LlmVerifyPayload extends Omit<StrategyEvaluationAiVerifyContext, 'evaluation'> {
llmGuidance: {
matchRateNote: string;
conditionTypeNote: string;
doNotMisinterpret: string[];
};
evaluation: Omit<StrategyEvaluationAiVerifyContext['evaluation'], 'buyConditions' | 'sellConditions'> & {
buyConditions: LlmConditionEvaluationRow[];
sellConditions: LlmConditionEvaluationRow[];
};
}
function walkConditionIndex(
node: LogicNode | null | undefined,
timeframe: string,
side: 'buy' | 'sell',
out: Map<string, ConditionDSL>,
seen: Set<string>,
): void {
if (!node) return;
if (node.type === 'TIMEFRAME') {
const tf = normalizeStartCandleType(node.candleTypes?.[0] ?? node.candleType ?? timeframe);
node.children?.forEach(c => walkConditionIndex(c, tf, side, out, seen));
return;
}
if (node.type === 'NOT') {
const child = node.children?.[0] ?? (node as LogicNode & { child?: LogicNode }).child;
if (child) walkConditionIndex(child, timeframe, side, out, seen);
return;
}
if (node.type === 'AND' || node.type === 'OR') {
node.children?.forEach(c => walkConditionIndex(c, timeframe, side, out, seen));
return;
}
if ((!node.type || node.type === 'CONDITION') && node.condition && node.id) {
const rowId = `${node.id}-${side}`;
const key = `${rowId}:${JSON.stringify(node.condition)}`;
if (seen.has(key)) return;
seen.add(key);
out.set(rowId, node.condition);
}
}
export function indexStrategyConditions(
buyRoot: unknown,
sellRoot: unknown,
): Map<string, ConditionDSL> {
const map = new Map<string, ConditionDSL>();
walkConditionIndex(asLogicNode(buyRoot), '1m', 'buy', map, new Set());
walkConditionIndex(asLogicNode(sellRoot), '1m', 'sell', map, new Set());
return map;
}
function formatRightOperand(cond: ConditionDSL, targetValue: number | null): string {
const rf = cond.rightField ?? '';
if (!rf || rf === 'NONE') return '—';
if (rf.startsWith('K_')) return rf;
if (isThresholdSymbol(rf)) {
const role = HL_ROLE_KO[rf] ?? rf;
const v = targetValue != null ? formatIndicatorValue(targetValue) : '?';
return `${rf}(${v}, ${role})`;
}
return rf;
}
/** LLM·로그용 — conditionType을 GT/LT로 축약하지 않음 */
export function formatConditionDslExpression(
cond: ConditionDSL,
timeframe: string,
targetValue: number | null,
): string {
const left = cond.leftField && cond.leftField !== 'none' ? cond.leftField : cond.indicatorType;
const right = formatRightOperand(cond, targetValue);
const tf = normalizeStartCandleType(cond.leftCandleType ?? cond.rightCandleType ?? timeframe);
const ctLabel = CONDITION_LABEL[cond.conditionType] ?? cond.conditionType;
return `[${tf}] ${left} ${cond.conditionType}(${ctLabel}) ${right}`;
}
export function formatConditionEvaluationSemantics(conditionType: string): string {
return EVALUATION_SEMANTICS[conditionType]
?? `Ta4j Rule.isSatisfied — ${CONDITION_LABEL[conditionType] ?? conditionType}`;
}
export function enrichConditionRow(
row: StrategyEvaluationAiVerifyContext['evaluation']['buyConditions'][number],
dslById: Map<string, ConditionDSL>,
): LlmConditionEvaluationRow {
const dsl = dslById.get(row.id);
if (!dsl) {
const fallback: ConditionDSL = {
indicatorType: row.indicatorType,
conditionType: row.conditionType,
leftField: row.indicatorType,
rightField: row.targetValue != null ? `K_${row.targetValue}` : undefined,
};
return {
id: row.id,
side: row.side as 'buy' | 'sell',
timeframe: row.timeframe,
indicatorType: row.indicatorType,
conditionType: row.conditionType,
satisfied: row.satisfied,
currentValue: row.currentValue,
dslCondition: fallback,
dslExpression: formatConditionDslExpression(fallback, row.timeframe, row.targetValue),
evaluationSemantics: formatConditionEvaluationSemantics(row.conditionType),
rightOperand: formatRightOperand(fallback, row.targetValue),
};
}
return {
id: row.id,
side: row.side as 'buy' | 'sell',
timeframe: row.timeframe,
indicatorType: row.indicatorType,
conditionType: row.conditionType,
satisfied: row.satisfied,
currentValue: row.currentValue,
dslCondition: dsl,
dslExpression: formatConditionDslExpression(dsl, row.timeframe, row.targetValue),
evaluationSemantics: formatConditionEvaluationSemantics(dsl.conditionType),
rightOperand: formatRightOperand(dsl, row.targetValue),
};
}
/** UI 컨텍스트 → LLM API 전송용 (오해 유발 thresholdLabel 제외) */
export function buildLlmVerifyPayload(ctx: StrategyEvaluationAiVerifyContext): LlmVerifyPayload {
const dslById = indexStrategyConditions(ctx.strategy.buyCondition, ctx.strategy.sellCondition);
return {
meta: ctx.meta,
strategy: {
id: ctx.strategy.id,
name: ctx.strategy.name,
buyCondition: ctx.strategy.buyCondition,
sellCondition: ctx.strategy.sellCondition,
},
chart: ctx.chart,
evaluation: {
loading: ctx.evaluation.loading,
error: ctx.evaluation.error,
matchRate: ctx.evaluation.matchRate,
overallEntryMet: ctx.evaluation.overallEntryMet,
overallExitMet: ctx.evaluation.overallExitMet,
buyConditions: ctx.evaluation.buyConditions.map(r => enrichConditionRow(r, dslById)),
sellConditions: ctx.evaluation.sellConditions.map(r => enrichConditionRow(r, dslById)),
},
signals: ctx.signals,
indicatorParams: ctx.indicatorParams,
deterministicHints: ctx.deterministicHints,
llmGuidance: {
matchRateNote:
'matchRate는 leaf 조건 satisfied=true 비율(0~100%)입니다. 0%는 모든 leaf가 미충족이라는 뜻이며, Rule↔시그널 불일치를 의미하지 않습니다.',
conditionTypeNote:
'CROSS_UP/CROSS_DOWN은 GT/LT(수준 비교)와 다릅니다. 교차는 직전봉·현재봉 2봉 비교 이벤트입니다.',
doNotMisinterpret: [
'evaluation.buyConditions/sellConditions의 dslExpression·evaluationSemantics를 기준으로 판단하세요.',
'thresholdLabel·"> 20" 형태 UI 표기는 LLM 컨텍스트에 포함하지 않았습니다.',
'deterministicHints.signalMatchesOverallRule이 "대체로 일치"이면 overall Rule과 차트 시그널은 일치합니다.',
'현재값>임계값인데 CROSS_UP 미충족은 정상일 수 있습니다(이미 돌파 후 유지).',
],
},
};
}
/** 수정 포인트·Cursor 프롬프트용 조건 한 줄 (DSL 기준) */
export function formatLlmConditionLine(row: LlmConditionEvaluationRow): string {
const status = row.satisfied === true ? '충족' : row.satisfied === false ? '미충족' : '평가불가';
const cur = row.currentValue != null ? formatIndicatorValue(row.currentValue) : '-';
return [
`- [${row.id}] ${row.dslExpression}`,
` Ta4j: ${row.evaluationSemantics}`,
` 결과: ${status} · 현재=${cur} · right=${row.rightOperand}`,
].join('\n');
}
export function formatLlmConditionLines(rows: LlmConditionEvaluationRow[]): string {
if (rows.length === 0) return '(조건 없음)';
return rows.map(formatLlmConditionLine).join('\n');
}