Files
dbx-main/app/modules/performance/db.py
T

449 lines
25 KiB
Python

"""performance_db PostgreSQL 저장소.
- 드라이버: psycopg 3 (`psycopg[binary,pool]`) — 다른 모듈과 동일 패턴.
- 연결 정보: 환경변수 `PERFORMANCE_DB_URL`
(예: postgresql://performance_app:<pwd>@postgres-db:5432/performance_db)
- 스키마는 앱이 만들지 않는다. `scripts/sql/performance_db_init.sql` 을 superuser 가 사전 적용.
- 연결 풀은 lazy open — 부팅 시 DB 가 잠시 끊겨도 컨테이너가 죽지 않게.
이 저장소는 인증정보를 **암호문 그대로만** 읽고 쓴다. 암·복호화는 credentials.py 의 몫.
"""
from __future__ import annotations
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import Any, Iterable
from psycopg import errors as pg_errors
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool
from app.timezone import to_kst_iso
# running 상태로 이 시간 넘게 남은 동기화는 죽은 것으로 보고 정리한다(서버 재시작 등).
STALE_RUN_AFTER = timedelta(minutes=30)
class ManualMigrationNeeded(Exception):
"""수동 입력(마이그레이션 002: perf_marketplaces.manual_input · perf_manual_sales)이 아직 적용되지 않음."""
def _iso(v: Any) -> Any:
if isinstance(v, datetime):
return to_kst_iso(v)
if isinstance(v, date):
return v.isoformat()
return v
class PerformanceDBStore:
def __init__(self, dsn: str, *, min_size: int = 1, max_size: int = 5):
self._pool = ConnectionPool(
conninfo=dsn,
min_size=min_size,
max_size=max_size,
kwargs={"row_factory": dict_row, "autocommit": True},
open=False,
)
self._pool.open(wait=False)
def close(self) -> None:
self._pool.close()
# ════════════════════════════════════════════════════════════
# 쇼핑몰 기본 설정
# ════════════════════════════════════════════════════════════
def list_marketplaces(self) -> list[dict[str, Any]]:
with self._pool.connection() as conn:
rows = conn.execute("SELECT * FROM perf_marketplaces ORDER BY sort_order, code").fetchall()
return [{k: _iso(v) for k, v in r.items()} for r in rows]
def update_marketplace(self, code: str, *, enabled: bool | None = None,
include_in_sales: bool | None = None,
manual_input: bool | None = None) -> dict[str, Any] | None:
sets, params = [], []
if manual_input is not None:
sets.append("manual_input = %s")
params.append(bool(manual_input))
if enabled is not None:
sets.append("enabled = %s")
params.append(bool(enabled))
if include_in_sales is not None:
sets.append("include_in_sales = %s")
params.append(bool(include_in_sales))
if not sets:
return None
params.append(code)
try:
with self._pool.connection() as conn:
row = conn.execute(
f"UPDATE perf_marketplaces SET {', '.join(sets)} WHERE code = %s RETURNING *", params
).fetchone()
except pg_errors.UndefinedColumn:
raise ManualMigrationNeeded() from None
return {k: _iso(v) for k, v in row.items()} if row else None
# ════════════════════════════════════════════════════════════
# 수수료율 이력
# ════════════════════════════════════════════════════════════
def rate_table(self) -> dict[str, list[tuple[date, Decimal]]]:
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT marketplace_code, effective_from, rate_percent FROM perf_commission_rates "
"WHERE rule_type = 'marketplace' ORDER BY marketplace_code, effective_from"
).fetchall()
out: dict[str, list[tuple[date, Decimal]]] = {}
for r in rows:
out.setdefault(r["marketplace_code"], []).append((r["effective_from"], Decimal(r["rate_percent"])))
return out
def list_rates(self, code: str) -> list[dict[str, Any]]:
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT id, rate_percent, effective_from, memo, created_by, created_at "
"FROM perf_commission_rates WHERE marketplace_code = %s AND rule_type = 'marketplace' "
"ORDER BY effective_from DESC",
(code,),
).fetchall()
return [{k: (str(v) if isinstance(v, Decimal) else _iso(v)) for k, v in r.items()} for r in rows]
def set_rate(self, code: str, *, rate_percent: Decimal, effective_from: date,
memo: str = "", user: str = "") -> None:
"""같은 적용 시작일이면 덮어쓴다(오타 수정), 다르면 이력으로 추가."""
with self._pool.connection() as conn:
conn.execute(
"""
INSERT INTO perf_commission_rates
(marketplace_code, rule_type, rate_percent, effective_from, memo, created_by)
VALUES (%s, 'marketplace', %s, %s, %s, %s)
ON CONFLICT (marketplace_code, rule_type, effective_from) DO UPDATE
SET rate_percent = EXCLUDED.rate_percent, memo = EXCLUDED.memo,
created_by = EXCLUDED.created_by, created_at = now()
""",
(code, rate_percent, effective_from, memo, user),
)
def delete_rate(self, code: str, rate_id: int) -> bool:
with self._pool.connection() as conn:
cur = conn.execute(
"DELETE FROM perf_commission_rates WHERE id = %s AND marketplace_code = %s", (rate_id, code)
)
return cur.rowcount > 0
# ════════════════════════════════════════════════════════════
# 인증정보(암호문)
# ════════════════════════════════════════════════════════════
def credential_rows(self, code: str) -> dict[str, dict[str, Any]]:
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT credential_key, encrypted_value, key_version, updated_at, updated_by "
"FROM perf_credentials WHERE marketplace_code = %s",
(code,),
).fetchall()
return {r["credential_key"]: r for r in rows}
def credential_keys_by_code(self) -> dict[str, set[str]]:
with self._pool.connection() as conn:
rows = conn.execute("SELECT marketplace_code, credential_key FROM perf_credentials").fetchall()
out: dict[str, set[str]] = {}
for r in rows:
out.setdefault(r["marketplace_code"], set()).add(r["credential_key"])
return out
def upsert_credential(self, code: str, key: str, *, encrypted_value: str, key_version: int,
user: str = "") -> None:
with self._pool.connection() as conn:
conn.execute(
"""
INSERT INTO perf_credentials (marketplace_code, credential_key, encrypted_value, key_version, updated_by)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (marketplace_code, credential_key) DO UPDATE
SET encrypted_value = EXCLUDED.encrypted_value, key_version = EXCLUDED.key_version,
updated_by = EXCLUDED.updated_by
""",
(code, key, encrypted_value, key_version, user),
)
def delete_credential(self, code: str, key: str) -> None:
with self._pool.connection() as conn:
conn.execute(
"DELETE FROM perf_credentials WHERE marketplace_code = %s AND credential_key = %s", (code, key)
)
def all_credential_rows(self) -> list[dict[str, Any]]:
with self._pool.connection() as conn:
return conn.execute(
"SELECT marketplace_code, credential_key, encrypted_value, key_version FROM perf_credentials"
).fetchall()
# ════════════════════════════════════════════════════════════
# 연결 상태
# ════════════════════════════════════════════════════════════
def statuses(self) -> dict[str, dict[str, Any]]:
with self._pool.connection() as conn:
rows = conn.execute("SELECT * FROM perf_connection_status").fetchall()
return {r["marketplace_code"]: {k: _iso(v) for k, v in r.items()} for r in rows}
def record_test(self, code: str, *, status: str, ok: bool, error: str = "") -> None:
with self._pool.connection() as conn:
conn.execute(
"""
INSERT INTO perf_connection_status (marketplace_code, status, last_test_at, last_test_ok,
last_error, last_error_at)
VALUES (%s, %s, now(), %s, %s, CASE WHEN %s = '' THEN NULL ELSE now() END)
ON CONFLICT (marketplace_code) DO UPDATE
SET status = EXCLUDED.status, last_test_at = now(), last_test_ok = EXCLUDED.last_test_ok,
last_error = CASE WHEN EXCLUDED.last_error = '' THEN perf_connection_status.last_error
ELSE EXCLUDED.last_error END,
last_error_at = CASE WHEN EXCLUDED.last_error = '' THEN perf_connection_status.last_error_at
ELSE now() END
""",
(code, status, ok, error, error),
)
def record_sync(self, code: str, *, status: str, ok: bool, error: str = "") -> None:
with self._pool.connection() as conn:
conn.execute(
"""
INSERT INTO perf_connection_status (marketplace_code, status, last_sync_at,
last_success_sync_at, last_error, last_error_at)
VALUES (%s, %s, now(), CASE WHEN %s THEN now() END, %s, CASE WHEN %s = '' THEN NULL ELSE now() END)
ON CONFLICT (marketplace_code) DO UPDATE
SET status = EXCLUDED.status, last_sync_at = now(),
last_success_sync_at = CASE WHEN %s THEN now()
ELSE perf_connection_status.last_success_sync_at END,
last_error = CASE WHEN EXCLUDED.last_error = '' THEN perf_connection_status.last_error
ELSE EXCLUDED.last_error END,
last_error_at = CASE WHEN EXCLUDED.last_error = '' THEN perf_connection_status.last_error_at
ELSE now() END
""",
(code, status, ok, error, error, ok),
)
# ════════════════════════════════════════════════════════════
# 매출 사건
# ════════════════════════════════════════════════════════════
def upsert_events(self, code: str, events: Iterable[dict[str, Any]], *, is_sample: bool = False) -> int:
"""같은 (쇼핑몰, 사건 종류, 외부 사건 ID)는 덮어쓴다 — 여러 번 동기화해도 중복 없음."""
rows = self._event_rows(code, events, is_sample)
if not rows:
return 0
with self._pool.connection() as conn:
with conn.transaction():
with conn.cursor() as cur:
cur.executemany(self._UPSERT_SQL, rows)
return len(rows)
def replace_events(self, code: str, start: date, end: date, events: Iterable[dict[str, Any]],
*, id_prefix: str = "") -> int:
"""기간(sales_date) 안 이 어댑터가 만든 실데이터(is_sample 아님)를 지우고 새로 넣는다 — 한 트랜잭션.
지우는 대상: 이 쇼핑몰(code)의 행 + (id_prefix 가 있으면) 다른 쇼핑몰 열로 보냈던 이 어댑터의 행
(external_event_id 가 id_prefix 로 시작). 다른 쇼핑몰의 직접 연동분·테스트 데이터는 건드리지 않는다.
실패하면 롤백되어 예전 데이터가 그대로 남는다."""
rows = self._event_rows(code, events, False)
with self._pool.connection() as conn:
with conn.transaction():
if id_prefix:
conn.execute(
"DELETE FROM perf_sales_events WHERE is_sample = FALSE AND sales_date BETWEEN %s AND %s "
"AND (marketplace_code = %s OR left(external_event_id, %s) = %s)",
(start, end, code, len(id_prefix), id_prefix),
)
else:
conn.execute(
"DELETE FROM perf_sales_events WHERE marketplace_code = %s AND is_sample = FALSE "
"AND sales_date BETWEEN %s AND %s",
(code, start, end),
)
if rows:
with conn.cursor() as cur:
cur.executemany(self._UPSERT_SQL, rows)
return len(rows)
@staticmethod
def _event_rows(code: str, events: Iterable[dict[str, Any]], is_sample: bool) -> list[tuple]:
"""행마다 집계할 쇼핑몰 = 사건의 marketplace_code(다른 쇼핑몰 대신 받은 주문), 없으면 code."""
return [
(e.get("marketplace_code") or code, e["event_type"], e["external_event_id"], e["external_order_id"],
e.get("external_line_id", ""),
e.get("order_date"), e.get("payment_date"), e["event_date"], e["sales_date"],
e.get("product_name", ""), e.get("option_name", ""), int(e.get("quantity") or 0),
int(e["amount"]), int(e.get("discount_amount") or 0), e.get("order_status", ""),
Jsonb(e.get("raw") or {}), is_sample)
for e in events
]
_UPSERT_SQL = """
INSERT INTO perf_sales_events
(marketplace_code, event_type, external_event_id, external_order_id, external_line_id,
order_date, payment_date, event_date, sales_date, product_name, option_name, quantity,
amount, discount_amount, order_status, raw_data, is_sample, synced_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now())
ON CONFLICT (marketplace_code, event_type, external_event_id) DO UPDATE
SET external_order_id = EXCLUDED.external_order_id, external_line_id = EXCLUDED.external_line_id,
order_date = EXCLUDED.order_date, payment_date = EXCLUDED.payment_date,
event_date = EXCLUDED.event_date, sales_date = EXCLUDED.sales_date,
product_name = EXCLUDED.product_name, option_name = EXCLUDED.option_name,
quantity = EXCLUDED.quantity, amount = EXCLUDED.amount,
discount_amount = EXCLUDED.discount_amount, order_status = EXCLUDED.order_status,
raw_data = EXCLUDED.raw_data, is_sample = EXCLUDED.is_sample, synced_at = now()
"""
def daily_amounts(self, start: date, end: date, codes: list[str]) -> dict[tuple[date, str], int]:
"""날짜 × 쇼핑몰 총매출. 수동 입력이 켜진 쇼핑몰은 수동 값을 넣은 날 그 값이 API 수집값을 대신한다
(실적 화면·회의록 머리 카드가 같은 숫자를 보게 여기서 합친다)."""
if not codes:
return {}
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT sales_date, marketplace_code, SUM(amount)::BIGINT AS amount "
"FROM perf_sales_events WHERE sales_date BETWEEN %s AND %s AND marketplace_code = ANY(%s) "
"GROUP BY sales_date, marketplace_code",
(start, end, codes),
).fetchall()
out = {(r["sales_date"], r["marketplace_code"]): int(r["amount"] or 0) for r in rows}
out.update(self.manual_amounts(start, end, codes))
return out
# ════════════════════════════════════════════════════════════
# 수동 입력 매출(마이그레이션 002)
# ════════════════════════════════════════════════════════════
def manual_amounts(self, start: date, end: date, codes: list[str], *,
active_only: bool = True) -> dict[tuple[date, str], int]:
"""수동 입력 값. active_only = 그 쇼핑몰 수동 입력이 켜진 것만(집계에 쓰는 값). 002 미적용이면 비어 있음."""
if not codes:
return {}
try:
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT s.sales_date, s.marketplace_code, s.amount FROM perf_manual_sales s "
"JOIN perf_marketplaces m ON m.code = s.marketplace_code "
"WHERE s.sales_date BETWEEN %s AND %s AND s.marketplace_code = ANY(%s)"
+ (" AND m.manual_input" if active_only else ""),
(start, end, codes),
).fetchall()
except (pg_errors.UndefinedTable, pg_errors.UndefinedColumn):
return {}
return {(r["sales_date"], r["marketplace_code"]): int(r["amount"]) for r in rows}
def api_amount(self, code: str, day: date) -> int:
"""그날 API 수집 합계(수동 값과 무관) — 수동 입력 칸에 참고로 보여 준다."""
with self._pool.connection() as conn:
row = conn.execute(
"SELECT COALESCE(SUM(amount), 0)::BIGINT AS amount FROM perf_sales_events "
"WHERE marketplace_code = %s AND sales_date = %s", (code, day),
).fetchone()
return int(row["amount"] or 0)
def set_manual_sale(self, code: str, day: date, amount: int | None, *, user: str = "") -> None:
"""수동 값 저장(amount None = 지움 → 그날은 다시 API 값)."""
try:
with self._pool.connection() as conn:
if amount is None:
conn.execute("DELETE FROM perf_manual_sales WHERE marketplace_code = %s AND sales_date = %s",
(code, day))
else:
conn.execute(
"INSERT INTO perf_manual_sales (marketplace_code, sales_date, amount, updated_by) "
"VALUES (%s, %s, %s, %s) ON CONFLICT (marketplace_code, sales_date) DO UPDATE "
"SET amount = EXCLUDED.amount, updated_by = EXCLUDED.updated_by",
(code, day, int(amount), user),
)
except pg_errors.UndefinedTable:
raise ManualMigrationNeeded() from None
def daily_breakdown(self, code: str, start: date, end: date, *, id_prefix: str = "") -> list[dict[str, Any]]:
"""쇼핑몰 화면(일별 매출내역) 대조용 — 날짜별 결제(SALE 합·건수)·환불(그 밖, 양수)·마지막 수집 시각.
id_prefix 가 있으면 이 어댑터가 다른 쇼핑몰 열로 보낸 사건까지 합친다(카페24 화면 합계와 같은 범위)."""
where = "(marketplace_code = %s OR left(external_event_id, %s) = %s)" if id_prefix else "marketplace_code = %s"
params: tuple = (code, len(id_prefix), id_prefix) if id_prefix else (code,)
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT sales_date, "
" COALESCE(SUM(amount) FILTER (WHERE event_type = 'SALE'), 0)::BIGINT AS paid, "
" COUNT(*) FILTER (WHERE event_type = 'SALE')::INT AS orders, "
" COALESCE(-SUM(amount) FILTER (WHERE event_type <> 'SALE'), 0)::BIGINT AS refund, "
" MAX(synced_at) AS synced_at "
f"FROM perf_sales_events WHERE is_sample = FALSE AND sales_date BETWEEN %s AND %s AND {where} "
"GROUP BY sales_date ORDER BY sales_date",
(start, end, *params),
).fetchall()
return [{"date": r["sales_date"].isoformat(), "paid": int(r["paid"]), "orders": int(r["orders"]),
"refund": int(r["refund"]), "synced_at": _iso(r["synced_at"])} for r in rows]
def list_events(self, code: str, start: date, end: date, *, limit: int = 500) -> list[dict[str, Any]]:
"""금액 클릭 상세(Drill-down). raw_data 는 보내지 않는다."""
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT id, event_type, external_order_id, external_line_id, order_date, payment_date, "
" event_date, sales_date, product_name, option_name, quantity, amount, "
" discount_amount, order_status, is_sample "
"FROM perf_sales_events WHERE marketplace_code = %s AND sales_date BETWEEN %s AND %s "
"ORDER BY sales_date, external_order_id, event_type LIMIT %s",
(code, start, end, limit),
).fetchall()
return [{k: _iso(v) for k, v in r.items()} for r in rows]
def sample_count(self) -> int:
with self._pool.connection() as conn:
row = conn.execute("SELECT COUNT(*) AS n FROM perf_sales_events WHERE is_sample").fetchone()
return int(row["n"] or 0)
def delete_sample_events(self) -> int:
"""테스트 데이터만(is_sample) 지운다 — 조건이 있는 삭제."""
with self._pool.connection() as conn:
cur = conn.execute("DELETE FROM perf_sales_events WHERE is_sample = TRUE")
return cur.rowcount
# ════════════════════════════════════════════════════════════
# 동기화 실행(중복 실행 방지)
# ════════════════════════════════════════════════════════════
def start_run(self, code: str, *, trigger: str, range_from: date, range_to: date, user: str) -> int | None:
"""running 행을 만든다. 이미 돌고 있으면 None(부분 유니크 인덱스로 막힘)."""
import psycopg # noqa: WPS433
with self._pool.connection() as conn:
conn.execute(
"UPDATE perf_sync_runs SET status = 'failed', finished_at = now(), "
"error = '중단됨(시간 초과로 정리)' "
"WHERE marketplace_code = %s AND status = 'running' AND started_at < now() - %s",
(code, STALE_RUN_AFTER),
)
try:
row = conn.execute(
"INSERT INTO perf_sync_runs (marketplace_code, trigger, range_from, range_to, started_by) "
"VALUES (%s, %s, %s, %s, %s) RETURNING id",
(code, trigger, range_from, range_to, user),
).fetchone()
except psycopg.errors.UniqueViolation:
return None
return int(row["id"])
def finish_run(self, run_id: int, *, ok: bool, fetched: int = 0, upserted: int = 0, error: str = "") -> None:
with self._pool.connection() as conn:
conn.execute(
"UPDATE perf_sync_runs SET status = %s, fetched_count = %s, upserted_count = %s, "
"error = %s, finished_at = now() WHERE id = %s",
("success" if ok else "failed", fetched, upserted, error, run_id),
)
def last_update_at(self) -> str | None:
"""대시보드 "마지막 업데이트" — 성공한 동기화 중 가장 최근."""
with self._pool.connection() as conn:
row = conn.execute(
"SELECT MAX(finished_at) AS t FROM perf_sync_runs WHERE status = 'success'"
).fetchone()
return _iso(row["t"]) if row and row["t"] else None
def running_codes(self) -> list[str]:
with self._pool.connection() as conn:
rows = conn.execute(
"SELECT marketplace_code FROM perf_sync_runs WHERE status = 'running' "
"AND started_at >= now() - %s",
(STALE_RUN_AFTER,),
).fetchall()
return [r["marketplace_code"] for r in rows]