From 76b9622c96159398b94afbf0e64b412aca5bf71d Mon Sep 17 00:00:00 2001 From: hywznn Date: Tue, 11 Aug 2026 23:17:51 +0900 Subject: [PATCH 1/5] =?UTF-8?q?feat(observability):=20AI=20=EC=8B=A4?= =?UTF-8?q?=ED=96=89=20=EB=8B=A8=EA=B3=84=EB=B3=84=20=EB=A1=9C=EA=B7=B8?= =?UTF-8?q?=EC=99=80=20=EC=A7=80=ED=91=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PLAN, Slot 조회, ANALYZE, 결과 저장과 Renewal 구간의 소요시간을 구조화 로그와 Micrometer 지표로 기록합니다. 운영 프로필에서는 Prometheus 공개 체인이 활성화되지 않도록 제한합니다. --- build.gradle | 1 + .../AiAnalysisContinuationService.java | 31 +++- .../application/AiRunExecutionTelemetry.java | 160 ++++++++++++++++++ .../airun/application/AiRunService.java | 144 ++++++++++++---- .../server/common/config/SecurityConfig.java | 18 +- .../renewal/RenewalExecutionService.java | 6 +- .../renewal/RenewalExecutionTelemetry.java | 75 ++++++-- src/main/resources/application.yaml | 9 +- 8 files changed, 386 insertions(+), 58 deletions(-) create mode 100644 src/main/java/com/fowoco/server/airun/application/AiRunExecutionTelemetry.java diff --git a/build.gradle b/build.gradle index db14f25b..3ce9dd13 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation 'org.apache.poi:poi-ooxml:5.4.0' runtimeOnly 'com.h2database:h2' runtimeOnly 'org.flywaydb:flyway-database-postgresql' + runtimeOnly 'io.micrometer:micrometer-registry-prometheus' runtimeOnly 'org.postgresql:postgresql' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-actuator-test' diff --git a/src/main/java/com/fowoco/server/airun/application/AiAnalysisContinuationService.java b/src/main/java/com/fowoco/server/airun/application/AiAnalysisContinuationService.java index 77fc57c1..84ff31de 100644 --- a/src/main/java/com/fowoco/server/airun/application/AiAnalysisContinuationService.java +++ b/src/main/java/com/fowoco/server/airun/application/AiAnalysisContinuationService.java @@ -20,6 +20,10 @@ import java.util.Objects; import java.util.UUID; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Phase.ANALYZE; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.ANALYZE_RUNTIME_CALL; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.SLOT_RESOLUTION; + /** * Continues a validated CONTEXT_REQUIRED result without holding a database transaction open. * #24 wires this service to a durable AiAttempt implementation. @@ -31,11 +35,13 @@ public final class AiAnalysisContinuationService { private final AiSlotResolutionTransaction slotResolutionTransaction; private final AiAttemptStarter attemptStarter; private final AiRuntimeClient runtimeClient; + private final AiRunExecutionTelemetry telemetry; public AiAnalysisContinuationService( AiSlotResolutionTransaction slotResolutionTransaction, AiAttemptStarter attemptStarter, - AiRuntimeClient runtimeClient + AiRuntimeClient runtimeClient, + AiRunExecutionTelemetry telemetry ) { this.slotResolutionTransaction = Objects.requireNonNull( slotResolutionTransaction, @@ -43,6 +49,7 @@ public AiAnalysisContinuationService( ); this.attemptStarter = Objects.requireNonNull(attemptStarter, "attemptStarter must not be null"); this.runtimeClient = Objects.requireNonNull(runtimeClient, "runtimeClient must not be null"); + this.telemetry = Objects.requireNonNull(telemetry, "telemetry must not be null"); } public AiAnalysisContinuationResult continueAnalysis( @@ -59,10 +66,16 @@ public AiAnalysisContinuationResult continueAnalysis( Objects.requireNonNull(callContext, "callContext must not be null"); validateContinuation(previousRequest, previousResponse, completedContextRounds); - AiSlotResolution resolution = slotResolutionTransaction.resolve( - companyId, - previousRequest.requiredKnowledgeVersion(), - previousResponse.contextRequirement() + AiSlotResolution resolution = telemetry.measure( + previousRequest.requestId(), + previousRequest.attemptId(), + ANALYZE, + SLOT_RESOLUTION, + () -> slotResolutionTransaction.resolve( + companyId, + previousRequest.requiredKnowledgeVersion(), + previousResponse.contextRequirement() + ) ); validateSameWorker(previousRequest, resolution.worker()); @@ -88,7 +101,13 @@ public AiAnalysisContinuationResult continueAnalysis( remainingDeadlineMs, analyzeInput ); - AiAnalysisResponse response = runtimeClient.analyze(analyzeRequest, callContext); + AiAnalysisResponse response = telemetry.measure( + analyzeRequest.requestId(), + analyzeRequest.attemptId(), + ANALYZE, + ANALYZE_RUNTIME_CALL, + () -> runtimeClient.analyze(analyzeRequest, callContext) + ); return new AiAnalysisContinuationResult( attemptId, response, diff --git a/src/main/java/com/fowoco/server/airun/application/AiRunExecutionTelemetry.java b/src/main/java/com/fowoco/server/airun/application/AiRunExecutionTelemetry.java new file mode 100644 index 00000000..897d19cc --- /dev/null +++ b/src/main/java/com/fowoco/server/airun/application/AiRunExecutionTelemetry.java @@ -0,0 +1,160 @@ +package com.fowoco.server.airun.application; + +import com.fowoco.server.aiintegration.application.error.AiRuntimeCallException; +import com.fowoco.server.aiintegration.application.error.AiRuntimeContractException; +import com.fowoco.server.aiintegration.application.model.AiAnalysisOutcome; +import com.fowoco.server.airun.application.error.AiContextResolutionException; +import com.fowoco.server.common.error.ApiException; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * Records only Server-observable AiRun stages. Instructions, Slot values and Runtime response + * bodies must never be included in logs or metric tags. + */ +@Component +final class AiRunExecutionTelemetry { + + private static final Logger log = LoggerFactory.getLogger(AiRunExecutionTelemetry.class); + + private final MeterRegistry meterRegistry; + + AiRunExecutionTelemetry(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + T measure( + UUID requestId, + UUID attemptId, + Phase phase, + Stage stage, + Supplier action + ) { + long startedNanos = System.nanoTime(); + try { + T result = action.get(); + long elapsedNanos = elapsedNanos(startedNanos); + recordStage(phase, stage, Status.SUCCESS, elapsedNanos); + log.info( + "event=ai_run_stage request_id={} attempt_id={} phase={} stage={} " + + "status=SUCCESS duration_ms={}", + requestId, + attemptId, + phase, + stage, + toMillis(elapsedNanos) + ); + return result; + } catch (RuntimeException exception) { + long elapsedNanos = elapsedNanos(startedNanos); + String failureCode = errorCode(exception); + recordStage(phase, stage, Status.FAILED, elapsedNanos); + recordFailure(phase, stage, failureCode); + log.warn( + "event=ai_run_stage request_id={} attempt_id={} phase={} stage={} " + + "status=FAILED duration_ms={} error_code={}", + requestId, + attemptId, + phase, + stage, + toMillis(elapsedNanos), + failureCode + ); + throw exception; + } + } + + void recordOutcome(AiAnalysisOutcome outcome) { + try { + Counter.builder("fowoco.ai.analysis.outcomes") + .description("Validated AI analysis response outcomes") + .tag("outcome", outcome.name()) + .register(meterRegistry) + .increment(); + } catch (RuntimeException metricFailure) { + log.warn("event=ai_run_metric status=FAILED metric=analysis_outcomes"); + } + } + + private void recordStage(Phase phase, Stage stage, Status status, long elapsedNanos) { + try { + Timer.builder("fowoco.ai.pipeline.stage") + .description("Server-observed AiRun pipeline stage duration") + .tag("phase", phase.name()) + .tag("stage", stage.name()) + .tag("status", status.name()) + .register(meterRegistry) + .record(elapsedNanos, TimeUnit.NANOSECONDS); + } catch (RuntimeException metricFailure) { + log.warn( + "event=ai_run_metric status=FAILED metric=pipeline_stage phase={} stage={}", + phase, + stage + ); + } + } + + private void recordFailure(Phase phase, Stage stage, String failureCode) { + try { + Counter.builder("fowoco.ai.pipeline.failures") + .description("Server-observed AI pipeline stage failures") + .tag("phase", phase.name()) + .tag("stage", stage.name()) + .tag("failure_code", failureCode) + .register(meterRegistry) + .increment(); + } catch (RuntimeException metricFailure) { + log.warn("event=ai_run_metric status=FAILED metric=pipeline_failures"); + } + } + + private long elapsedNanos(long startedNanos) { + return Math.max(0L, System.nanoTime() - startedNanos); + } + + private long toMillis(long elapsedNanos) { + return TimeUnit.NANOSECONDS.toMillis(elapsedNanos); + } + + private String errorCode(RuntimeException exception) { + if (exception instanceof AiRuntimeCallException runtimeFailure) { + return runtimeFailure.failureCode().name(); + } + if (exception instanceof AiRuntimeContractException contractFailure) { + return contractFailure.failureCode().name(); + } + if (exception instanceof AiContextResolutionException contextFailure) { + return contextFailure.failureCode().name(); + } + if (exception instanceof ApiException apiFailure) { + return apiFailure.errorCode().code(); + } + return "UNEXPECTED_AI_RUN_FAILURE"; + } + + enum Phase { + PIPELINE, + PLAN, + ANALYZE + } + + enum Stage { + PLAN_RUNTIME_CALL, + SLOT_RESOLUTION, + ANALYZE_RUNTIME_CALL, + RESULT_PERSIST, + TOTAL + } + + private enum Status { + SUCCESS, + FAILED + } +} diff --git a/src/main/java/com/fowoco/server/airun/application/AiRunService.java b/src/main/java/com/fowoco/server/airun/application/AiRunService.java index 1f547df9..ad2f5509 100644 --- a/src/main/java/com/fowoco/server/airun/application/AiRunService.java +++ b/src/main/java/com/fowoco/server/airun/application/AiRunService.java @@ -51,6 +51,14 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Phase.ANALYZE; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Phase.PIPELINE; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Phase.PLAN; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.ANALYZE_RUNTIME_CALL; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.PLAN_RUNTIME_CALL; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.RESULT_PERSIST; +import static com.fowoco.server.airun.application.AiRunExecutionTelemetry.Stage.TOTAL; + /** * Owns the demo vertical slice: persist first, call Runtime without a database transaction, * and persist the validated result in a new transaction. @@ -76,6 +84,7 @@ public class AiRunService implements AiAttemptStarter { private final AuditEventRepository auditEventRepository; private final AiRunPublicEventPublisher publicEventPublisher; private final Executor aiRunTaskExecutor; + private final AiRunExecutionTelemetry telemetry; public AiRunService( ActorAuthorizer actorAuthorizer, @@ -90,7 +99,8 @@ public AiRunService( TransactionTemplate transactionTemplate, AuditEventRepository auditEventRepository, AiRunPublicEventPublisher publicEventPublisher, - @Qualifier("aiRunTaskExecutor") Executor aiRunTaskExecutor + @Qualifier("aiRunTaskExecutor") Executor aiRunTaskExecutor, + AiRunExecutionTelemetry telemetry ) { this.actorAuthorizer = actorAuthorizer; this.tenantDatabaseContext = tenantDatabaseContext; @@ -105,6 +115,7 @@ public AiRunService( this.auditEventRepository = auditEventRepository; this.publicEventPublisher = publicEventPublisher; this.aiRunTaskExecutor = aiRunTaskExecutor; + this.telemetry = telemetry; } public AiRunResult createAndSchedule( @@ -315,36 +326,81 @@ private void executePlan(AiRunCreation creation) { clock.instant() )); publishCurrent(initial.aiRunId(), initial.companyId()); - AiAnalysisResponse planResponse = runtimeClient.analyze( - creation.request(), - AiRuntimeCallContext.withoutTrace() + telemetry.measure( + creation.request().requestId(), + creation.request().attemptId(), + PIPELINE, + TOTAL, + () -> { + executePlanMeasured(creation); + return null; + } ); - saveSuccess(creation.aiRunId(), creation.companyId(), creation.request().attemptId(), planResponse); - if (planResponse.outcome() == AiAnalysisOutcome.CONTEXT_REQUIRED) { - AiAnalysisContinuationResult result = new AiAnalysisContinuationService( - slotResolutionTransaction, - this, - runtimeClient - ).continueAnalysis( - creation.companyId(), - creation.request(), - planResponse, - 0, - runtimeDeadlinePolicy.attemptDeadlineMs(), - AiRuntimeCallContext.withoutTrace() - ); - saveSuccess( - creation.aiRunId(), - creation.companyId(), - result.attemptId(), - result.response() - ); - } } catch (RuntimeException exception) { markLatestFailed(initial, exception); } } + private void executePlanMeasured(AiRunCreation creation) { + AiAnalysisResponse planResponse = telemetry.measure( + creation.request().requestId(), + creation.request().attemptId(), + PLAN, + PLAN_RUNTIME_CALL, + () -> runtimeClient.analyze( + creation.request(), + AiRuntimeCallContext.withoutTrace() + ) + ); + telemetry.measure( + creation.request().requestId(), + creation.request().attemptId(), + PLAN, + RESULT_PERSIST, + () -> { + saveSuccess( + creation.aiRunId(), + creation.companyId(), + creation.request().attemptId(), + planResponse + ); + return null; + } + ); + if (planResponse.outcome() != AiAnalysisOutcome.CONTEXT_REQUIRED) { + return; + } + + AiAnalysisContinuationResult result = new AiAnalysisContinuationService( + slotResolutionTransaction, + this, + runtimeClient, + telemetry + ).continueAnalysis( + creation.companyId(), + creation.request(), + planResponse, + 0, + runtimeDeadlinePolicy.attemptDeadlineMs(), + AiRuntimeCallContext.withoutTrace() + ); + telemetry.measure( + creation.request().requestId(), + result.attemptId(), + ANALYZE, + RESULT_PERSIST, + () -> { + saveSuccess( + creation.aiRunId(), + creation.companyId(), + result.attemptId(), + result.response() + ); + return null; + } + ); + } + private void schedulePlan(AiRunCreation creation) { try { aiRunTaskExecutor.execute(() -> executePlan(creation)); @@ -358,11 +414,40 @@ private void schedulePlan(AiRunCreation creation) { private void executeOne(ExecutionState state, AiAnalysisRequest request) { try { - AiAnalysisResponse response = runtimeClient.analyze( - request, - AiRuntimeCallContext.withoutTrace() + telemetry.measure( + request.requestId(), + request.attemptId(), + ANALYZE, + TOTAL, + () -> { + AiAnalysisResponse response = telemetry.measure( + request.requestId(), + request.attemptId(), + ANALYZE, + ANALYZE_RUNTIME_CALL, + () -> runtimeClient.analyze( + request, + AiRuntimeCallContext.withoutTrace() + ) + ); + telemetry.measure( + request.requestId(), + request.attemptId(), + ANALYZE, + RESULT_PERSIST, + () -> { + saveSuccess( + state.aiRunId(), + state.companyId(), + request.attemptId(), + response + ); + return null; + } + ); + return null; + } ); - saveSuccess(state.aiRunId(), state.companyId(), request.attemptId(), response); } catch (RuntimeException exception) { markLatestFailed(state, exception); } @@ -378,6 +463,7 @@ private void saveSuccess( repository.markAttemptSucceeded(aiRunId, companyId, attemptId, response, clock.instant()); return null; }); + telemetry.recordOutcome(response.outcome()); publishCurrent(aiRunId, companyId); } diff --git a/src/main/java/com/fowoco/server/common/config/SecurityConfig.java b/src/main/java/com/fowoco/server/common/config/SecurityConfig.java index b4dfb1ef..4797307b 100644 --- a/src/main/java/com/fowoco/server/common/config/SecurityConfig.java +++ b/src/main/java/com/fowoco/server/common/config/SecurityConfig.java @@ -38,6 +38,22 @@ public UserDetailsService emptyUserDetailsService() { @Bean @Order(1) + @Profile("observability & !prod") + public SecurityFilterChain prometheusSecurityFilterChain(HttpSecurity http) throws Exception { + http + .securityMatcher("/actuator/prometheus") + .authorizeHttpRequests(authorize -> authorize.anyRequest().permitAll()) + .csrf(csrf -> csrf.disable()) + .requestCache(cache -> cache.disable()) + .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) + .formLogin(form -> form.disable()) + .httpBasic(basic -> basic.disable()) + .logout(logout -> logout.disable()); + return http.build(); + } + + @Bean + @Order(2) @Profile("local") public SecurityFilterChain h2ConsoleSecurityFilterChain(HttpSecurity http) throws Exception { http @@ -49,7 +65,7 @@ public SecurityFilterChain h2ConsoleSecurityFilterChain(HttpSecurity http) throw } @Bean - @Order(2) + @Order(3) public SecurityFilterChain applicationSecurityFilterChain( HttpSecurity http, @Qualifier("handlerExceptionResolver") HandlerExceptionResolver exceptionResolver, diff --git a/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionService.java b/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionService.java index 1fcd702a..8b5878fd 100644 --- a/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionService.java +++ b/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionService.java @@ -56,7 +56,7 @@ public RenewalExecutionResult execute( ) { UUID runtimeRequestId = uuidGenerator.generate(); UUID attemptId = uuidGenerator.generate(); - return telemetry.measure(runtimeRequestId, metadata.requestId(), taskId, TOTAL, () -> + return telemetry.measure(runtimeRequestId, metadata.requestId(), TOTAL, () -> executeMeasured(taskId, command, actor, metadata, runtimeRequestId, attemptId) ); } @@ -72,7 +72,6 @@ private RenewalExecutionResult executeMeasured( RenewalExecutionContext context = telemetry.measure( runtimeRequestId, metadata.requestId(), - taskId, CONTEXT_LOAD, () -> contextReader.load( taskId, @@ -99,7 +98,6 @@ private RenewalExecutionResult executeMeasured( RenewalRunResponse response = telemetry.measure( runtimeRequestId, metadata.requestId(), - taskId, RENEWAL_RUNTIME_CALL, () -> runtimeClient.run(request, AiRuntimeCallContext.withoutTrace()) ); @@ -108,7 +106,6 @@ private RenewalExecutionResult executeMeasured( ? telemetry.measure( runtimeRequestId, metadata.requestId(), - taskId, DOCUMENT_GENERATION, () -> generatedDocumentService.prepare(response.generatedDocuments()) ) @@ -116,7 +113,6 @@ private RenewalExecutionResult executeMeasured( return telemetry.measure( runtimeRequestId, metadata.requestId(), - taskId, RESULT_APPLY, () -> resultApplier.apply( taskId, diff --git a/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetry.java b/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetry.java index 06600a45..528b3901 100644 --- a/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetry.java +++ b/src/main/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetry.java @@ -3,6 +3,9 @@ import com.fowoco.server.aiintegration.application.error.AiRuntimeCallException; import com.fowoco.server.aiintegration.application.error.AiRuntimeContractException; import com.fowoco.server.common.error.ApiException; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -19,48 +22,85 @@ final class RenewalExecutionTelemetry { private static final Logger log = LoggerFactory.getLogger(RenewalExecutionTelemetry.class); + private final MeterRegistry meterRegistry; + + RenewalExecutionTelemetry(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + T measure( UUID requestId, String httpRequestId, - UUID taskId, Stage stage, Supplier action ) { long startedNanos = System.nanoTime(); try { T result = action.get(); + long elapsedNanos = elapsedNanos(startedNanos); + recordStage(stage, Status.SUCCESS, elapsedNanos); log.info( - "event=renewal_execution_stage request_id={} http_request_id={} " - + "task_id={} stage={} " + "event=renewal_execution_stage request_id={} http_request_id={} stage={} " + "status=SUCCESS duration_ms={}", requestId, httpRequestId, - taskId, stage, - elapsedMillis(startedNanos) + toMillis(elapsedNanos) ); return result; } catch (RuntimeException exception) { + long elapsedNanos = elapsedNanos(startedNanos); + String failureCode = errorCode(exception); + recordStage(stage, Status.FAILED, elapsedNanos); + recordFailure(stage, failureCode); log.warn( - "event=renewal_execution_stage request_id={} http_request_id={} " - + "task_id={} stage={} " + "event=renewal_execution_stage request_id={} http_request_id={} stage={} " + "status=FAILED duration_ms={} error_code={}", requestId, httpRequestId, - taskId, stage, - elapsedMillis(startedNanos), - errorCode(exception) + toMillis(elapsedNanos), + failureCode ); throw exception; } } - private long elapsedMillis(long startedNanos) { - return Math.max( - 0L, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos) - ); + private void recordStage(Stage stage, Status status, long elapsedNanos) { + try { + Timer.builder("fowoco.renewal.stage") + .description("Server-observed Renewal execution stage duration") + .tag("stage", stage.name()) + .tag("status", status.name()) + .register(meterRegistry) + .record(elapsedNanos, TimeUnit.NANOSECONDS); + } catch (RuntimeException metricFailure) { + log.warn( + "event=renewal_metric status=FAILED metric=execution_stage stage={}", + stage + ); + } + } + + private void recordFailure(Stage stage, String failureCode) { + try { + Counter.builder("fowoco.renewal.failures") + .description("Server-observed Renewal execution stage failures") + .tag("stage", stage.name()) + .tag("failure_code", failureCode) + .register(meterRegistry) + .increment(); + } catch (RuntimeException metricFailure) { + log.warn("event=renewal_metric status=FAILED metric=failures"); + } + } + + private long elapsedNanos(long startedNanos) { + return Math.max(0L, System.nanoTime() - startedNanos); + } + + private long toMillis(long elapsedNanos) { + return TimeUnit.NANOSECONDS.toMillis(elapsedNanos); } private String errorCode(RuntimeException exception) { @@ -83,4 +123,9 @@ enum Stage { RESULT_APPLY, TOTAL } + + private enum Status { + SUCCESS, + FAILED + } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index bca69f14..a401745c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -29,7 +29,7 @@ management: endpoints: web: exposure: - include: health + include: health,prometheus endpoint: health: show-details: never @@ -45,6 +45,11 @@ management: enabled: true readinessstate: enabled: true + metrics: + distribution: + percentiles-histogram: + fowoco.ai.pipeline.stage: true + fowoco.renewal.stage: true server: shutdown: graceful @@ -187,7 +192,7 @@ spring: enabled: true path: /h2-console server: - address: 127.0.0.1 + address: ${SERVER_ADDRESS:127.0.0.1} app: auth: jwt: From 0924b2fdb1e10376b19ca9740a7d3de3df086b3d Mon Sep 17 00:00:00 2001 From: hywznn Date: Tue, 11 Aug 2026 23:18:01 +0900 Subject: [PATCH 2/5] =?UTF-8?q?test(observability):=20=EB=8B=A8=EA=B3=84?= =?UTF-8?q?=20=EC=A7=80=ED=91=9C=EC=99=80=20endpoint=20=EB=B3=B4=EC=95=88?= =?UTF-8?q?=20=EA=B2=80=EC=A6=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PLAN부터 ANALYZE까지의 단계 기록, OUT_OF_SCOPE 단축 흐름, 민감정보 없는 실패 로그와 Prometheus endpoint 공개 조건을 검증합니다. --- .../fowoco/server/ServerApplicationTests.java | 7 + .../server/airun/AiRunApiIntegrationTest.java | 40 ++++++ .../AiAnalysisContinuationServiceTest.java | 7 +- .../AiRunExecutionTelemetryTest.java | 124 ++++++++++++++++++ .../PrometheusEndpointIntegrationTest.java | 65 +++++++++ .../RenewalExecutionTelemetryTest.java | 45 ++++++- 6 files changed, 282 insertions(+), 6 deletions(-) create mode 100644 src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java create mode 100644 src/test/java/com/fowoco/server/airun/application/PrometheusEndpointIntegrationTest.java diff --git a/src/test/java/com/fowoco/server/ServerApplicationTests.java b/src/test/java/com/fowoco/server/ServerApplicationTests.java index 0df640ef..258ba0a7 100644 --- a/src/test/java/com/fowoco/server/ServerApplicationTests.java +++ b/src/test/java/com/fowoco/server/ServerApplicationTests.java @@ -36,6 +36,13 @@ void healthApiIsPublic() throws Exception { assertThat(response.headers().firstValue("X-Request-Id")).contains("test-request-001"); } + @Test + void prometheusEndpointIsNotPublicWithoutObservabilityProfile() throws Exception { + HttpResponse response = get("/actuator/prometheus"); + + assertThat(response.statusCode()).isEqualTo(401); + } + @Test void openApiIncludesHealthApi() throws Exception { HttpResponse response = get("/v3/api-docs"); diff --git a/src/test/java/com/fowoco/server/airun/AiRunApiIntegrationTest.java b/src/test/java/com/fowoco/server/airun/AiRunApiIntegrationTest.java index a521314c..1ce0707a 100644 --- a/src/test/java/com/fowoco/server/airun/AiRunApiIntegrationTest.java +++ b/src/test/java/com/fowoco/server/airun/AiRunApiIntegrationTest.java @@ -18,6 +18,8 @@ import com.fowoco.server.aiintegration.application.model.AnalysisInput; import com.fowoco.server.aiintegration.application.port.AiRuntimeClient; import com.jayway.jsonpath.JsonPath; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.io.InputStream; import java.math.BigDecimal; import java.net.URI; @@ -73,6 +75,9 @@ class AiRunApiIntegrationTest { @Autowired private ObjectMapper objectMapper; + @Autowired + private MeterRegistry meterRegistry; + @MockitoBean private AiRuntimeClient runtimeClient; @@ -123,6 +128,10 @@ void resetAndSeed() { @Test void createsQueriesAnswersAndFinishesOneWorkerAnalysis() throws Exception { + long planBefore = stageCount("PLAN", "PLAN_RUNTIME_CALL", "SUCCESS"); + long slotBefore = stageCount("ANALYZE", "SLOT_RESOLUTION", "SUCCESS"); + long analyzeBefore = stageCount("ANALYZE", "ANALYZE_RUNTIME_CALL", "SUCCESS"); + double reviewRequiredBefore = outcomeCount("REVIEW_REQUIRED"); String token = login(HR_A_EMAIL); HttpResponse created = post( "/api/v1/ai-runs", @@ -205,10 +214,21 @@ void createsQueriesAnswersAndFinishesOneWorkerAnalysis() throws Exception { verify(runtimeClient, atLeast(3)).analyze(requestCaptor.capture(), any()); assertThat(requestCaptor.getAllValues()) .allSatisfy(request -> assertThat(request.deadlineMs()).isEqualTo(240_000L)); + assertThat(stageCount("PLAN", "PLAN_RUNTIME_CALL", "SUCCESS") - planBefore) + .isEqualTo(1); + assertThat(stageCount("ANALYZE", "SLOT_RESOLUTION", "SUCCESS") - slotBefore) + .isEqualTo(1); + assertThat(stageCount("ANALYZE", "ANALYZE_RUNTIME_CALL", "SUCCESS") - analyzeBefore) + .isEqualTo(2); + assertThat(outcomeCount("REVIEW_REQUIRED") - reviewRequiredBefore) + .isEqualTo(1.0); } @Test void finishesOutOfScopeAfterPlanWithoutResolvingSlotsOrCallingAnalyze() throws Exception { + long slotBefore = stageCount("ANALYZE", "SLOT_RESOLUTION", "SUCCESS"); + long analyzeBefore = stageCount("ANALYZE", "ANALYZE_RUNTIME_CALL", "SUCCESS"); + double outOfScopeBefore = outcomeCount("OUT_OF_SCOPE"); reset(runtimeClient); runtimeCalls.set(0); when(runtimeClient.analyze(any(), any())).thenAnswer(invocation -> { @@ -255,6 +275,26 @@ void finishesOutOfScopeAfterPlanWithoutResolvingSlotsOrCallingAnalyze() throws E assertThat(events.body()) .contains("event:COMPLETED", "\"analysis_outcome\":\"OUT_OF_SCOPE\"") .doesNotContain("event:SLOT_CHECKING"); + assertThat(stageCount("ANALYZE", "SLOT_RESOLUTION", "SUCCESS")) + .isEqualTo(slotBefore); + assertThat(stageCount("ANALYZE", "ANALYZE_RUNTIME_CALL", "SUCCESS")) + .isEqualTo(analyzeBefore); + assertThat(outcomeCount("OUT_OF_SCOPE") - outOfScopeBefore) + .isEqualTo(1.0); + } + + private long stageCount(String phase, String stage, String status) { + Timer timer = meterRegistry.find("fowoco.ai.pipeline.stage") + .tags("phase", phase, "stage", stage, "status", status) + .timer(); + return timer == null ? 0L : timer.count(); + } + + private double outcomeCount(String outcome) { + var counter = meterRegistry.find("fowoco.ai.analysis.outcomes") + .tag("outcome", outcome) + .counter(); + return counter == null ? 0.0 : counter.count(); } @Test diff --git a/src/test/java/com/fowoco/server/airun/application/AiAnalysisContinuationServiceTest.java b/src/test/java/com/fowoco/server/airun/application/AiAnalysisContinuationServiceTest.java index 32c2a608..a0994d89 100644 --- a/src/test/java/com/fowoco/server/airun/application/AiAnalysisContinuationServiceTest.java +++ b/src/test/java/com/fowoco/server/airun/application/AiAnalysisContinuationServiceTest.java @@ -24,6 +24,7 @@ import com.fowoco.server.workflow.application.WorkflowCatalogService; import com.fowoco.server.workflow.domain.WorkflowCatalog; import com.fowoco.server.workflow.domain.WorkflowDefinition; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; @@ -72,7 +73,8 @@ void recordsNewAttemptBeforeAnalyzeAndPreservesPlanContext() { callOrder.add("attempt"); return NEXT_ATTEMPT_ID; }, - validatingClient + validatingClient, + new AiRunExecutionTelemetry(new SimpleMeterRegistry()) ); AiAnalysisContinuationResult result = service.continueAnalysis( @@ -134,7 +136,8 @@ void stopsAtRoundLimitBeforeDatabaseResolutionOrRuntimeCall() { }, (request, context) -> { throw new AssertionError("Runtime must not be called after the round limit."); - } + }, + new AiRunExecutionTelemetry(new SimpleMeterRegistry()) ); assertThatThrownBy(() -> service.continueAnalysis( diff --git a/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java b/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java new file mode 100644 index 00000000..69e71336 --- /dev/null +++ b/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java @@ -0,0 +1,124 @@ +package com.fowoco.server.airun.application; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import com.fowoco.server.aiintegration.application.error.AiRuntimeCallException; +import com.fowoco.server.aiintegration.application.error.AiRuntimeFailureCode; +import com.fowoco.server.aiintegration.application.model.AiAnalysisOutcome; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +class AiRunExecutionTelemetryTest { + + private static final UUID REQUEST_ID = UUID.fromString( + "10000000-0000-0000-0000-000000000001" + ); + private static final UUID ATTEMPT_ID = UUID.fromString( + "20000000-0000-0000-0000-000000000001" + ); + private static final String SENSITIVE_FAILURE_MESSAGE = + "NGUYEN VAN AN passport_number=M12345678 instruction=체류연장"; + + private final Logger logger = (Logger) LoggerFactory.getLogger(AiRunExecutionTelemetry.class); + private final ListAppender appender = new ListAppender<>(); + private final SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + private final AiRunExecutionTelemetry telemetry = new AiRunExecutionTelemetry(meterRegistry); + + @BeforeEach + void attachAppender() { + appender.start(); + logger.addAppender(appender); + } + + @AfterEach + void detachAppender() { + logger.detachAppender(appender); + appender.stop(); + meterRegistry.close(); + } + + @Test + void successRecordsSafeStructuredLogAndTimer() { + String result = telemetry.measure( + REQUEST_ID, + ATTEMPT_ID, + AiRunExecutionTelemetry.Phase.PLAN, + AiRunExecutionTelemetry.Stage.PLAN_RUNTIME_CALL, + () -> "ok" + ); + + assertThat(result).isEqualTo("ok"); + assertThat(appender.list).hasSize(1); + assertThat(appender.list.get(0).getFormattedMessage()) + .contains( + "request_id=" + REQUEST_ID, + "attempt_id=" + ATTEMPT_ID, + "phase=PLAN", + "stage=PLAN_RUNTIME_CALL", + "status=SUCCESS", + "duration_ms=" + ); + assertThat(meterRegistry.get("fowoco.ai.pipeline.stage") + .tag("phase", "PLAN") + .tag("stage", "PLAN_RUNTIME_CALL") + .tag("status", "SUCCESS") + .timer() + .count()).isEqualTo(1); + } + + @Test + void failureRecordsOnlyStableCodeAndNeverSensitiveMessage() { + assertThatThrownBy(() -> telemetry.measure( + REQUEST_ID, + ATTEMPT_ID, + AiRunExecutionTelemetry.Phase.ANALYZE, + AiRunExecutionTelemetry.Stage.ANALYZE_RUNTIME_CALL, + () -> { + throw new AiRuntimeCallException( + AiRuntimeFailureCode.DEADLINE_EXCEEDED, + SENSITIVE_FAILURE_MESSAGE + ); + } + )).isInstanceOf(AiRuntimeCallException.class); + + assertThat(appender.list).hasSize(1); + assertThat(appender.list.get(0).getFormattedMessage()) + .contains( + "phase=ANALYZE", + "stage=ANALYZE_RUNTIME_CALL", + "status=FAILED", + "error_code=DEADLINE_EXCEEDED" + ) + .doesNotContain( + "NGUYEN VAN AN", + "passport_number", + "M12345678", + "체류연장" + ); + assertThat(appender.list.get(0).getThrowableProxy()).isNull(); + assertThat(meterRegistry.get("fowoco.ai.pipeline.failures") + .tag("phase", "ANALYZE") + .tag("stage", "ANALYZE_RUNTIME_CALL") + .tag("failure_code", "DEADLINE_EXCEEDED") + .counter() + .count()).isEqualTo(1); + } + + @Test + void outcomeCounterUsesOnlyBoundedOutcomeTag() { + telemetry.recordOutcome(AiAnalysisOutcome.REVIEW_REQUIRED); + + assertThat(meterRegistry.get("fowoco.ai.analysis.outcomes") + .tag("outcome", "REVIEW_REQUIRED") + .counter() + .count()).isEqualTo(1); + } +} diff --git a/src/test/java/com/fowoco/server/airun/application/PrometheusEndpointIntegrationTest.java b/src/test/java/com/fowoco/server/airun/application/PrometheusEndpointIntegrationTest.java new file mode 100644 index 00000000..508f8488 --- /dev/null +++ b/src/test/java/com/fowoco/server/airun/application/PrometheusEndpointIntegrationTest.java @@ -0,0 +1,65 @@ +package com.fowoco.server.airun.application; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.boot.micrometer.metrics.test.autoconfigure.AutoConfigureMetrics; +import org.springframework.test.context.ActiveProfiles; + +@ActiveProfiles({"test", "observability"}) +@AutoConfigureMetrics +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class PrometheusEndpointIntegrationTest { + + private static final UUID REQUEST_ID = UUID.fromString( + "10000000-0000-0000-0000-000000000001" + ); + private static final UUID ATTEMPT_ID = UUID.fromString( + "20000000-0000-0000-0000-000000000001" + ); + + @LocalServerPort + private int port; + + @Autowired + private AiRunExecutionTelemetry telemetry; + + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @Test + void observabilityProfileExposesPrometheusMetricsWithoutBusinessAuthentication() throws Exception { + telemetry.measure( + REQUEST_ID, + ATTEMPT_ID, + AiRunExecutionTelemetry.Phase.PLAN, + AiRunExecutionTelemetry.Stage.PLAN_RUNTIME_CALL, + () -> "measured" + ); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + port + "/actuator/prometheus")) + .GET() + .build(); + HttpResponse response = httpClient.send( + request, + HttpResponse.BodyHandlers.ofString() + ); + + assertThat(response.statusCode()).isEqualTo(200); + assertThat(response.body()) + .contains( + "fowoco_ai_pipeline_stage_seconds_count", + "phase=\"PLAN\"", + "stage=\"PLAN_RUNTIME_CALL\"", + "status=\"SUCCESS\"" + ); + } +} diff --git a/src/test/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetryTest.java b/src/test/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetryTest.java index 6a69beea..b3870daa 100644 --- a/src/test/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetryTest.java +++ b/src/test/java/com/fowoco/server/task/application/renewal/RenewalExecutionTelemetryTest.java @@ -8,6 +8,7 @@ import ch.qos.logback.core.read.ListAppender; import com.fowoco.server.aiintegration.application.error.AiRuntimeCallException; import com.fowoco.server.aiintegration.application.error.AiRuntimeFailureCode; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.util.UUID; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -28,7 +29,8 @@ class RenewalExecutionTelemetryTest { private final Logger logger = (Logger) LoggerFactory.getLogger(RenewalExecutionTelemetry.class); private final ListAppender appender = new ListAppender<>(); - private final RenewalExecutionTelemetry telemetry = new RenewalExecutionTelemetry(); + private final SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + private final RenewalExecutionTelemetry telemetry = new RenewalExecutionTelemetry(meterRegistry); @BeforeEach void attachAppender() { @@ -47,7 +49,6 @@ void failureLogContainsOnlyStableCodeAndNeverTheExceptionMessage() { assertThatThrownBy(() -> telemetry.measure( REQUEST_ID, HTTP_REQUEST_ID, - TASK_ID, RenewalExecutionTelemetry.Stage.RENEWAL_RUNTIME_CALL, () -> { throw new AiRuntimeCallException( @@ -63,12 +64,48 @@ void failureLogContainsOnlyStableCodeAndNeverTheExceptionMessage() { .contains( "request_id=" + REQUEST_ID, "http_request_id=" + HTTP_REQUEST_ID, - "task_id=" + TASK_ID, "stage=RENEWAL_RUNTIME_CALL", "status=FAILED", "error_code=DEADLINE_EXCEEDED" ) - .doesNotContain("NGUYEN VAN AN", "passport_number", "M12345678"); + .doesNotContain( + "task_id", + TASK_ID.toString(), + "NGUYEN VAN AN", + "passport_number", + "M12345678" + ); assertThat(event.getThrowableProxy()).isNull(); + assertThat(meterRegistry.get("fowoco.renewal.stage") + .tag("stage", "RENEWAL_RUNTIME_CALL") + .tag("status", "FAILED") + .timer() + .count()).isEqualTo(1); + assertThat(meterRegistry.get("fowoco.renewal.failures") + .tag("stage", "RENEWAL_RUNTIME_CALL") + .tag("failure_code", "DEADLINE_EXCEEDED") + .counter() + .count()).isEqualTo(1); + } + + @Test + void successRecordsDurationWithoutResourceIdentifiers() { + String result = telemetry.measure( + REQUEST_ID, + HTTP_REQUEST_ID, + RenewalExecutionTelemetry.Stage.CONTEXT_LOAD, + () -> "loaded" + ); + + assertThat(result).isEqualTo("loaded"); + assertThat(appender.list).hasSize(1); + assertThat(appender.list.get(0).getFormattedMessage()) + .contains("stage=CONTEXT_LOAD", "status=SUCCESS", "duration_ms=") + .doesNotContain("task_id", TASK_ID.toString()); + assertThat(meterRegistry.get("fowoco.renewal.stage") + .tag("stage", "CONTEXT_LOAD") + .tag("status", "SUCCESS") + .timer() + .count()).isEqualTo(1); } } From 9e76181ce6fed5ab980d2f86714977f9453b007c Mon Sep 17 00:00:00 2001 From: hywznn Date: Tue, 11 Aug 2026 23:18:09 +0900 Subject: [PATCH 3/5] =?UTF-8?q?docs(observability):=20=EB=A1=9C=EC=BB=AC?= =?UTF-8?q?=20Prometheus=20=EC=B8=A1=EC=A0=95=20=EA=B0=80=EC=9D=B4?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 로컬 Compose, scrape 설정, PromQL 예시와 반복 측정 기준을 문서화해 데모 정량 평가에 재사용할 수 있게 합니다. --- README.md | 1 + compose.observability.yml | 20 ++++ docs/ai-pipeline-observability.md | 176 ++++++++++++++++++++++++++++++ prometheus/prometheus.yml | 10 ++ 4 files changed, 207 insertions(+) create mode 100644 compose.observability.yml create mode 100644 docs/ai-pipeline-observability.md create mode 100644 prometheus/prometheus.yml diff --git a/README.md b/README.md index bdb36ab0..74c51a4d 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,7 @@ src/main/java/com/fowoco/server/ | Server ↔ AI 계약 | [AI Runtime 계약](docs/ai-runtime-contract.md) | Server가 AI에 보내고 받을 수 있는 값과 검증 기준 | | 근로자 명단 가져오기 | [Worker Import 가이드](docs/worker-import.md) | CSV/XLSX 업로드부터 검증·수정·등록까지의 API 순서 | | Agent DB 정보 보충 | [Slot 조회·재호출](docs/ai-slot-resolution.md) | canonical key allow-list, tenant 조회와 ANALYZE 재호출 기준 | +| AI 단계별 성능 측정 | [AI 파이프라인 관측·Prometheus 가이드](docs/ai-pipeline-observability.md) | PLAN·Slot·ANALYZE·Renewal 구간의 정량 평가와 로컬 Prometheus 확인 기준 | | 이벤트 유실·재처리 | [Outbox 운영 가이드](docs/reliability/transactional-outbox.md) | 이벤트 발행, lease, 재시도와 장애 복구 기준 | | 구현 계획·업무 상태 | [Server Roadmap](https://github.com/orgs/fowoco/projects/3) · [Issues](https://github.com/fowoco/server/issues) | 실제 담당자, 우선순위와 진행 상태 | | 전체 설명·운영 가이드 | [Server Wiki](https://github.com/fowoco/server/wiki) | 초보자용 아키텍처·API·배포 설명 | diff --git a/compose.observability.yml b/compose.observability.yml new file mode 100644 index 00000000..31a15e36 --- /dev/null +++ b/compose.observability.yml @@ -0,0 +1,20 @@ +name: fowoco-server-observability + +services: + prometheus: + image: prom/prometheus:v3.13.2 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-lifecycle + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - fowoco-prometheus-data:/prometheus + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + fowoco-prometheus-data: diff --git a/docs/ai-pipeline-observability.md b/docs/ai-pipeline-observability.md new file mode 100644 index 00000000..678cf8f5 --- /dev/null +++ b/docs/ai-pipeline-observability.md @@ -0,0 +1,176 @@ +# AI 파이프라인 소요시간 측정 가이드 + +## 목적 + +이 문서는 FOWOCO의 `PLAN → Slot 조회 → ANALYZE → Renewal → 문서 생성` 흐름을 +Server 경계에서 반복 측정하고, 데모 발표의 정량 평가 근거로 정리하는 방법을 +설명합니다. + +Server는 AI Runtime HTTP 왕복시간을 측정합니다. BERT·A.X·LangGraph 내부 Node의 +실행시간과 모델 정확도는 AI 팀의 평가 결과를 사용합니다. 두 값을 같은 지표로 +혼합하지 않습니다. + +## 기록되는 단계 + +### AiRun + +| Phase | Stage | 의미 | +| --- | --- | --- | +| `PLAN` | `PLAN_RUNTIME_CALL` | AI Runtime의 PLAN 요청·응답과 Server 계약 검증 | +| `ANALYZE` | `SLOT_RESOLUTION` | tenant와 allow-list를 적용한 DB 업무정보 조회 | +| `ANALYZE` | `ANALYZE_RUNTIME_CALL` | PLAN 결정을 재사용한 ANALYZE 요청·응답과 검증 | +| `PLAN/ANALYZE` | `RESULT_PERSIST` | 검증된 결과 저장과 공개 상태 Event 발행 | +| `PIPELINE/ANALYZE` | `TOTAL` | 자동 실행 묶음의 전체 시간. HR 답변 대기시간은 포함하지 않음 | + +### Renewal + +| Stage | 의미 | +| --- | --- | +| `CONTEXT_LOAD` | Task·Worker·Company·OCR·Slot Context 조회 | +| `RENEWAL_RUNTIME_CALL` | Renewal Agent HTTP 왕복과 계약 검증 | +| `DOCUMENT_GENERATION` | HWP/HWPX 생성 결과 준비 | +| `RESULT_APPLY` | 안내 초안·Task 상태·생성 파일 반영 | +| `TOTAL` | Renewal 자동 실행 전체 시간 | + +구조화 로그에는 추적용 `request_id`, `attempt_id`, 단계, 성공·실패, 소요시간과 +안전한 오류 코드만 남습니다. 발화문, 실명, Slot 실제 값, Token과 AI 응답 본문은 +남기지 않습니다. + +## 로컬 실행 + +### 1. Server 실행 + +Prometheus endpoint는 기본적으로 인증 없이 접근할 수 없습니다. 로컬 측정할 때만 +`observability` profile을 함께 활성화합니다. + +```bash +SERVER_ADDRESS=0.0.0.0 SPRING_PROFILES_ACTIVE=local,observability ./gradlew bootRun +``` + +Docker Desktop의 Prometheus가 호스트의 Server에 접근하도록 로컬 실행 주소만 +`0.0.0.0`으로 변경합니다. 신뢰할 수 있는 개발 네트워크에서만 사용하고 측정 후 +Server를 종료합니다. 이 값을 주지 않으면 Server는 계속 `127.0.0.1`에만 바인딩됩니다. + +다음 주소에서 Prometheus 형식의 원시 지표를 확인합니다. + +```text +http://127.0.0.1:8080/actuator/prometheus +``` + +`observability` profile은 `prod`와 함께 활성화해도 공개 Security Chain이 생성되지 +않도록 막혀 있습니다. 운영 환경에서 Prometheus를 연결할 때는 별도 내부망 인증 +정책을 먼저 정합니다. + +### 2. Prometheus 실행 + +Server가 실행된 상태에서 다음 명령을 사용합니다. + +```bash +docker compose -f compose.observability.yml up -d +``` + +- Prometheus: http://127.0.0.1:9090 +- 수집 상태: http://127.0.0.1:9090/targets + +측정을 마치고 컨테이너만 중지할 때는 다음을 사용합니다. + +```bash +docker compose -f compose.observability.yml down +``` + +로컬 측정 데이터까지 삭제하려면 명시적으로 `down -v`를 사용합니다. + +## 주요 Metric + +| Metric | 설명 | +| --- | --- | +| `fowoco_ai_pipeline_stage_seconds` | AiRun 단계별 Server 관측시간 | +| `fowoco_ai_analysis_outcomes_total` | 검증된 분석 응답 Outcome 횟수 | +| `fowoco_ai_pipeline_failures_total` | 단계와 안전한 오류 코드별 실패 횟수 | +| `fowoco_renewal_stage_seconds` | Renewal 단계별 Server 관측시간 | +| `fowoco_renewal_failures_total` | Renewal 오류 코드별 실패 횟수 | + +Timer는 Prometheus에서 `_count`, `_sum`, `_max`, `_bucket` 시계열로 노출됩니다. +Prometheus 시간 단위는 초이고 구조화 로그의 `duration_ms`는 밀리초입니다. + +## PromQL 예시 + +최근 30분 PLAN 평균시간: + +```promql +sum(increase(fowoco_ai_pipeline_stage_seconds_sum{phase="PLAN",stage="PLAN_RUNTIME_CALL",status="SUCCESS"}[30m])) +/ +sum(increase(fowoco_ai_pipeline_stage_seconds_count{phase="PLAN",stage="PLAN_RUNTIME_CALL",status="SUCCESS"}[30m])) +``` + +최근 30분 단계별 중앙값: + +```promql +histogram_quantile( + 0.50, + sum by (le, phase, stage) ( + increase(fowoco_ai_pipeline_stage_seconds_bucket{status="SUCCESS"}[30m]) + ) +) +``` + +최근 30분 단계별 95백분위: + +```promql +histogram_quantile( + 0.95, + sum by (le, phase, stage) ( + increase(fowoco_ai_pipeline_stage_seconds_bucket{status="SUCCESS"}[30m]) + ) +) +``` + +분석 결과별 횟수: + +```promql +sum by (outcome) (fowoco_ai_analysis_outcomes_total) +``` + +오류 코드별 실패 단계 횟수: + +```promql +sum by (phase, stage, failure_code) (fowoco_ai_pipeline_failures_total) +``` + +한 번의 오류는 실제 실패 단계와 이를 감싼 `TOTAL` 단계에 각각 기록될 수 있습니다. +실패한 요청 수만 확인할 때는 `stage="TOTAL"`을, 원인 단계를 확인할 때는 +`stage!="TOTAL"`을 사용합니다. + +## 정량 평가 절차 + +1. Server와 AI Runtime을 기동합니다. +2. 모델 최초 로딩을 위한 워밍업 요청을 2회 실행하고 결과에서 제외합니다. +3. 동일한 합성 발화와 Demo Worker를 사용해 시나리오별 최소 10회 실행합니다. +4. 각 실행의 `request_id`, cold/warm 여부, Outcome과 모델 버전을 별도 표에 기록합니다. +5. Prometheus에서 단계별 중앙값·최댓값·실패 횟수를 조회합니다. +6. AI 팀의 모델 내부 추론시간·정확도와 Server E2E 시간을 구분해 보고합니다. + +권장 시나리오: + +| 시나리오 | 확인할 결과 | +| --- | --- | +| 정상 체류연장 | `PLAN → Slot → ANALYZE → REVIEW_REQUIRED` | +| 누락 정보 | `NEEDS_INFO → HR 답변 → ANALYZE` | +| 범위 밖 발화 | `OUT_OF_SCOPE`, Slot·ANALYZE 미실행 | +| Runtime 지연 | 240초 제한과 `DEADLINE_EXCEEDED` | +| Renewal 문서 생성 | Context·Runtime·문서 생성·결과 반영 | + +평가표 예시: + +| 시나리오 | 반복 | PLAN | Slot | ANALYZE | 전체 | Outcome | 비고 | +| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | +| 정상 체류연장 | 1 | 0.84s | 0.02s | 0.41s | 1.34s | REVIEW_REQUIRED | warm | + +Server 지표만으로 `HR 업무시간이 몇 % 절감됐다`고 결론 내리지 않습니다. 이 효과를 +제시하려면 동일 업무의 수작업 시간과 FOWOCO 사용시간을 별도로 측정해야 합니다. + +## 보안과 Metric tag + +허용 tag는 `phase`, `stage`, `status`, `outcome`, `failure_code`처럼 값의 종류가 +제한된 항목뿐입니다. `requestId`, `attemptId`, `companyId`, `workerId`, `taskId`, +실명과 연락처를 Metric tag에 넣지 않습니다. 추적 ID는 구조화 로그에서만 사용합니다. diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 00000000..c8efb4d1 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: fowoco-server-local + metrics_path: /actuator/prometheus + static_configs: + - targets: + - host.docker.internal:8080 From aacbaac89aaa20a74f14392e58eac882791fb289 Mon Sep 17 00:00:00 2001 From: hywznn Date: Tue, 11 Aug 2026 23:20:13 +0900 Subject: [PATCH 4/5] =?UTF-8?q?test(observability):=20=EA=B3=84=EC=95=BD?= =?UTF-8?q?=20=EC=98=A4=EB=A5=98=20=EC=A7=80=ED=91=9C=20=EA=B5=AC=EB=B6=84?= =?UTF-8?q?=20=EA=B2=80=EC=A6=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runtime timeout과 응답 계약 오류가 서로 다른 안전한 failure_code로 기록되는지 확인합니다. --- .../AiRunExecutionTelemetryTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java b/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java index 69e71336..736f33bf 100644 --- a/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java +++ b/src/test/java/com/fowoco/server/airun/application/AiRunExecutionTelemetryTest.java @@ -7,6 +7,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import com.fowoco.server.aiintegration.application.error.AiRuntimeCallException; +import com.fowoco.server.aiintegration.application.error.AiRuntimeContractException; import com.fowoco.server.aiintegration.application.error.AiRuntimeFailureCode; import com.fowoco.server.aiintegration.application.model.AiAnalysisOutcome; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -112,6 +113,32 @@ void failureRecordsOnlyStableCodeAndNeverSensitiveMessage() { .count()).isEqualTo(1); } + @Test + void contractFailureIsSeparatedFromRuntimeTimeout() { + assertThatThrownBy(() -> telemetry.measure( + REQUEST_ID, + ATTEMPT_ID, + AiRunExecutionTelemetry.Phase.PLAN, + AiRunExecutionTelemetry.Stage.PLAN_RUNTIME_CALL, + () -> { + throw new AiRuntimeContractException( + AiRuntimeFailureCode.INVALID_RESPONSE_CONTRACT, + SENSITIVE_FAILURE_MESSAGE + ); + } + )).isInstanceOf(AiRuntimeContractException.class); + + assertThat(meterRegistry.get("fowoco.ai.pipeline.failures") + .tag("phase", "PLAN") + .tag("stage", "PLAN_RUNTIME_CALL") + .tag("failure_code", "INVALID_RESPONSE_CONTRACT") + .counter() + .count()).isEqualTo(1); + assertThat(appender.list.get(0).getFormattedMessage()) + .contains("error_code=INVALID_RESPONSE_CONTRACT") + .doesNotContain(SENSITIVE_FAILURE_MESSAGE); + } + @Test void outcomeCounterUsesOnlyBoundedOutcomeTag() { telemetry.recordOutcome(AiAnalysisOutcome.REVIEW_REQUIRED); From 8a031d7205e9c2ac2e0dac297a71811f20e35dd0 Mon Sep 17 00:00:00 2001 From: hywznn Date: Tue, 11 Aug 2026 23:35:26 +0900 Subject: [PATCH 5/5] =?UTF-8?q?docs(observability):=20=EA=B0=9C=EB=B0=9C?= =?UTF-8?q?=C2=B7=EB=B0=B0=ED=8F=AC=20=EB=AC=B8=EC=84=9C=EC=97=90=20?= =?UTF-8?q?=EA=B4=80=EC=B8=A1=20=EA=B8=B0=EC=A4=80=20=EC=97=B0=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 로컬 Prometheus 실행법, 운영 공개 금지 원칙과 Server·Infra의 관측 소유권을 기존 가이드에 반영합니다. --- docs/deployment-runbook.md | 19 +++++++++++++++++++ docs/development-guide.md | 24 ++++++++++++++++++++++++ docs/project-structure.md | 8 ++++++++ 3 files changed, 51 insertions(+) diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index 30ce423f..891cf55e 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -72,6 +72,24 @@ SMTP 비밀번호와 재설정 원본 token은 Git, Issue, 일반 로그에 기 DB pool은 기본 최대 10개입니다. 클러스터 규모에 따라 `DB_MAX_POOL_SIZE`, `DB_MIN_IDLE`, `DB_CONNECTION_TIMEOUT_MS`, `DB_VALIDATION_TIMEOUT_MS`로 제한합니다. +## 관측 설정 경계 + +Server는 AiRun·Renewal 구간의 Micrometer 지표를 생성하지만, 현재 데모 배포에서는 +Prometheus를 클러스터에 함께 배포하지 않습니다. + +- `/actuator/prometheus`는 기본 보안 Chain에서 보호됩니다. +- 로컬 `observability` profile은 `prod`와 함께 활성화해도 공개 Chain이 생성되지 + 않습니다. +- 배포 환경에서 수집이 필요해지면 Infra가 내부 Service·NetworkPolicy·인증 또는 + 별도 management port를 먼저 구성합니다. +- 공개 Ingress와 `CORS_ALLOWED_ORIGINS`에 Prometheus endpoint를 추가하지 않습니다. +- Metric에는 `companyId`, `workerId`, `taskId`, 요청·시도 ID와 개인정보를 tag로 + 넣지 않습니다. + +따라서 현재 `server-env`에 `SPRING_PROFILES_ACTIVE=prod,observability`를 설정하면 +안 됩니다. 로컬 측정과 정량 평가 절차는 +[AI 파이프라인 관측 가이드](ai-pipeline-observability.md)를 사용합니다. + 현재 Infra에 HTTPS/TLS와 `RELEASED` Workflow Catalog 배포가 없으면 `prod` 완료 조건을 충족하지 못합니다. 임시 HTTP 주소와 DRAFT Catalog는 개발 Smoke에만 사용합니다. @@ -143,6 +161,7 @@ Seed의 수량과 고정 ID도 첫 기동과 같아야 합니다. 7. Worker Link 대표 흐름 확인 8. SMS가 활성화된 환경에서는 실제 수신·링크 접속·중복 발송 방지 확인 9. SMTP가 활성화된 환경에서는 재설정 메일 수신·링크 token·새 비밀번호 로그인 확인 +10. 로그에서 AiRun·Renewal `TOTAL` 단계와 안전한 `failure_code`가 기록되는지 확인 Runtime 장애 테스트에서는 가짜 AI 결과를 만들지 않고 안전한 오류 또는 수동 처리 상태로 남아야 합니다. diff --git a/docs/development-guide.md b/docs/development-guide.md index b2ed1695..a86b89f3 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -117,6 +117,29 @@ Server는 AI에 보낼 수 있는 필드를 typed DTO로 제한하고 요청 전 상세 계약은 [AI Runtime 계약 문서](ai-runtime-contract.md)를 확인합니다. +## AI 파이프라인 로컬 관측 + +Server는 AI 내부 Node를 추측하지 않고 자신이 실제로 관측할 수 있는 구간만 +구조화 로그와 Micrometer 지표로 기록합니다. + +- AiRun: PLAN 호출, Slot 조회, ANALYZE 호출, 결과 저장, 전체 자동 실행 +- Renewal: Context 조회, Runtime 호출, 문서 생성, 결과 반영, 전체 실행 +- `request_id`와 `attempt_id`는 로그 추적에만 사용합니다. +- Metric tag에는 종류가 제한된 `phase`, `stage`, `status`, `outcome`, + `failure_code`만 사용합니다. +- 발화문, Slot 값, 근로자·Task ID와 개인정보는 로그 본문과 Metric tag에 넣지 + 않습니다. + +로컬 Prometheus를 함께 실행할 때는 다음과 같이 별도 profile을 명시합니다. + +```bash +SERVER_ADDRESS=0.0.0.0 SPRING_PROFILES_ACTIVE=local,observability ./gradlew bootRun +docker compose -f compose.observability.yml up -d +``` + +운영 `prod`에서는 위 공개 profile을 사용하지 않습니다. 단계 정의, PromQL과 반복 +측정 방법은 [AI 파이프라인 관측 가이드](ai-pipeline-observability.md)를 확인합니다. + ## 이벤트 유실 방지와 재처리 Task 생성·취소처럼 후속 처리가 필요한 변경은 업무 데이터와 @@ -270,5 +293,6 @@ Client가 안전한 형식의 `X-Request-Id`를 보내면 Server가 응답과 - [프로젝트 구조](project-structure.md) - [Transactional Outbox 운영 가이드](reliability/transactional-outbox.md) - [PostgreSQL Runtime Timeout 운영 가이드](reliability/postgresql-runtime-timeouts.md) +- [AI 파이프라인 관측·Prometheus 가이드](ai-pipeline-observability.md) - [ADR 목록](adr/README.md) - [PostgreSQL RLS 적용 가이드](database/postgresql-rls-rollout.md) diff --git a/docs/project-structure.md b/docs/project-structure.md index 6a4d457b..38521834 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -71,6 +71,11 @@ server/ | `aiintegration` | AI Runtime HTTP 계약과 Client | | `reliability` | Outbox, event 전달과 복구 | +관측 코드도 기능 소유권을 따릅니다. AiRun 단계 지표는 `airun`, Renewal 단계 +지표는 `task`가 기록합니다. `common`은 Actuator endpoint의 보안·공통 설정만 +담당하고, Prometheus의 배포·보존·알림은 `infra` 저장소가 소유합니다. 별도 +`observability` 도메인 패키지나 Metric 저장용 DB 테이블은 만들지 않습니다. + ## 기능 내부 구조 기능 코드가 커지면 아래 방향으로 확장합니다. @@ -115,6 +120,9 @@ PostgreSQL AI Runtime - `client`는 화면 상태와 사용자 상호작용을 소유합니다. - `infra`는 통합 배포, 네트워크, Secret과 관측 인프라를 소유합니다. +Server가 기록하는 단계와 Infra가 수집하는 범위는 +[AI 파이프라인 관측 가이드](ai-pipeline-observability.md)를 기준으로 맞춥니다. + 따라서 `server`의 `aiintegration`에는 Provider SDK나 Prompt Builder를 넣지 않습니다. `workflow`은 Knowledge projection을 읽지만 원본 정의를 수정하지 않습니다.