23주차 코드 뒤풀이: metric·requestId·DB wait부터 alert·runbook까지

metric,type,tags,meaning,cardinality,alert_or_dashboard
http_server_requests,timer,route|status|outcome,customer request rate error and duration,LOW,RED rate error and p95 dashboard
transfer_completed,counter,outcome,committed transfer outcomes by fixed category,LOW,transfer outcome dashboard
transfer_retry,counter,sqlstate_category,bounded database lock retry attempts,LOW,retry burst alert
reconciliation_mismatch,gauge,job,unresolved reconciliation mismatch count,LOW,ledger mismatch alert
idempotency_stale,gauge,operation,stale processing idempotency records,LOW,stale request alert

package com.example.financialcore.observability;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import java.util.Objects;
import java.util.function.Supplier;

public final class TransferMetrics {
    private final MeterRegistry registry;

    public TransferMetrics(MeterRegistry registry) {
        this.registry = Objects.requireNonNull(registry);
    }

    public <T> T record(Supplier<T> action) {
        Timer.Sample sample = Timer.start(registry);
        try {
            T result = action.get();
            registry.counter("fcl.transfer", "outcome", "completed").increment();
            return result;
        } catch (RuntimeException rejected) {
            registry.counter("fcl.transfer", "outcome", "business_rejected").increment();
            throw rejected;
        } finally {
            sample.stop(registry.timer("fcl.transfer.duration"));
        }
    }
}

package com.example.financialcore.observability;

import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.Test;

import static org.assertj.core.api.Assertions.*;

class TransferMetricsTest {
    @Test
    void success_records_fixed_outcome_and_duration() {
        var registry = new SimpleMeterRegistry();
        var metrics = new TransferMetrics(registry);
        assertThat(metrics.record(() -> "ok")).isEqualTo("ok");
        assertThat(registry.get("fcl.transfer").tag("outcome", "completed").counter().count()).isEqualTo(1.0);
        assertThat(registry.get("fcl.transfer.duration").timer().count()).isEqualTo(1);
    }

    @Test
    void rejection_records_fixed_outcome_and_still_stops_timer() {
        var registry = new SimpleMeterRegistry();
        var metrics = new TransferMetrics(registry);
        assertThatThrownBy(() -> metrics.record(() -> {
            throw new IllegalArgumentException("rejected");
        })).isInstanceOf(IllegalArgumentException.class);
        assertThat(registry.get("fcl.transfer").tag("outcome", "business_rejected").counter().count()).isEqualTo(1.0);
        assertThat(registry.get("fcl.transfer.duration").timer().count()).isEqualTo(1);
    }
}

# W23 incident runbook (provided fixture)

1. Preserve the inbound `X-Request-Id` from the application log and response.
2. Capture the exact failing request time, HTTP status, and sanitized account identifier.
3. Run the disposable `run-w23-db-wait.ps1` drill; never inspect an assumed shared database.
4. Attach `gate.json` and `db-wait.json` hashes. A runbook review alone never owns Green.
5. Escalate when the request ID is absent, the wait remains after rollback, or cleanup is nonzero.

param([Parameter(Mandatory=$true)][string]$ProjectRoot,[Parameter(Mandatory=$true)][string]$EvidenceDir)
$ErrorActionPreference='Stop';$root=(Resolve-Path -LiteralPath $ProjectRoot).Path;$e=[IO.Path]::GetFullPath($EvidenceDir);$class='com.example.financialcore.api.RequestIdFilterTest';$runbook=Join-Path $root 'evidence/w23/incident-runbook.md';if(!(Test-Path -LiteralPath $runbook -PathType Leaf)){throw 'W23 hashed runbook missing'};Push-Location $root
try{& .\gradlew.bat test --rerun-tasks --no-daemon --tests $class;if($LASTEXITCODE-ne 0){throw "W23 Gradle exit=$LASTEXITCODE"};$xml=Get-Item -LiteralPath 'build/test-results/test/TEST-com.example.financialcore.api.RequestIdFilterTest.xml';[xml]$x=Get-Content -Raw $xml.FullName;$s=$x.testsuite;if([string]$s.name-ne$class-or[int]$s.tests-ne 2-or[int]$s.failures-ne 0-or[int]$s.errors-ne 0-or[int]$s.skipped-ne 0){throw 'W23 exact RequestIdFilter XML mismatch'};New-Item -ItemType Directory -Force $e|Out-Null;$body=[ordered]@{classes=@($class);tests=2;failures=0;errors=0;skipped=0;claim='request-id-regression-and-runbook';runbook_sha256=(Get-FileHash -Algorithm SHA256 -LiteralPath $runbook).Hash.ToLowerInvariant();native_exit=0};$tmp=Join-Path $e 'gate.json.tmp';$body|ConvertTo-Json|Set-Content -Encoding utf8 $tmp;Move-Item -Force -LiteralPath $tmp -Destination (Join-Path $e 'gate.json');'W23_OBSERVABILITY_GREEN classes=1 tests=2 claim=request-id-regression-and-runbook native_exit=0'}finally{Pop-Location}

-- 학습용 예시 · 정본 답안 아님
-- 입력 grain: business_tx 1행/거래, ledger_entry 1행/원장 항목, audit_event 1행/감사 사건
-- 출력 grain: 거래 1행. 두 1:N 자식을 동시에 생으로 JOIN하면 N×M으로 행이 늘어날 수 있다.

SELECT tx.tx_id,
       COUNT(*) AS joined_rows,
       COUNT(DISTINCT le.entry_id) AS ledger_rows,
       COUNT(DISTINCT ae.audit_id) AS audit_rows
FROM business_tx AS tx
JOIN ledger_entry AS le ON le.tx_id = tx.tx_id
JOIN audit_event AS ae ON ae.tx_id = tx.tx_id
GROUP BY tx.tx_id
ORDER BY tx.tx_id;

-- 안전한 집계 예시: 각 1:N 자식을 거래 grain으로 먼저 줄인 뒤 JOIN한다.
WITH ledger_counts AS (
    SELECT tx_id, COUNT(*) AS ledger_rows
    FROM ledger_entry
    GROUP BY tx_id
),
audit_counts AS (
    SELECT tx_id, COUNT(*) AS audit_rows
    FROM audit_event
    WHERE tx_id IS NOT NULL
    GROUP BY tx_id
)
SELECT tx.tx_id,
       COALESCE(le.ledger_rows, 0) AS ledger_rows,
       COALESCE(ae.audit_rows, 0) AS audit_rows
FROM business_tx AS tx
LEFT JOIN ledger_counts AS le ON le.tx_id = tx.tx_id
LEFT JOIN audit_counts AS ae ON ae.tx_id = tx.tx_id
ORDER BY tx.tx_id;

-- 격리 반례: parent 1행 × child_a 2행 × child_b 3행 = JOIN 결과 6행.
WITH child_a(id) AS (VALUES (1), (2)),
child_b(id) AS (VALUES (10), (20), (30))
SELECT COUNT(*) AS multiplied_rows
FROM child_a
CROSS JOIN child_b;