Files
dbx-corm/tests/test_google_resilience.py

303 lines
13 KiB
Python

"""Fault-injection tests. All Google writes and DB connections are replaced."""
import asyncio
import copy
import json
import os
import tempfile
import threading
import time
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from urllib.parse import urlsplit
import requests
_import_cache_dir = tempfile.TemporaryDirectory()
os.environ["SETTINGS_CACHE_FILE"] = str(Path(_import_cache_dir.name) / "settings.json")
# 서버판(dbx-corm)은 ERP 중앙 인증으로 로그인 안 된 요청을 401 로 막는다.
# 테스트는 서비스 간 호출 토큰(X-Service-Token)으로 통과한다.
os.environ["INTERNAL_SERVICE_TOKEN"] = "test-service-token"
import main
import google_runtime as runtime
import routers.cs_records as records
ROWS = [["대분류 (Section)", "항목 (Key)", "설정값 1"], ["APP_SETTINGS", "theme", "old"]]
async def request(method, url, payload=None):
parsed = urlsplit(url)
body = b"" if payload is None else json.dumps(payload).encode()
events = []
received = False
async def receive():
nonlocal received
if not received:
received = True
return {"type": "http.request", "body": body, "more_body": False}
await asyncio.Event().wait()
async def send(event):
events.append(event)
await main.app({
"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1",
"method": method, "scheme": "http", "path": parsed.path,
"raw_path": parsed.path.encode(), "query_string": parsed.query.encode(),
"root_path": "", "server": ("test", 80), "client": ("127.0.0.1", 1234),
"headers": [(b"content-type", b"application/json"), (b"x-service-token", b"test-service-token")],
}, receive, send)
start = next(e for e in events if e["type"] == "http.response.start")
content = b"".join(e.get("body", b"") for e in events if e["type"] == "http.response.body")
return start["status"], dict(start["headers"]), content
async def wait_until(predicate):
async with asyncio.timeout(2):
while not predicate():
await asyncio.sleep(0.005)
class CacheTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
self.cache = runtime.SettingsCache(Path(self.directory.name) / "settings.json", "test", ttl=60)
def tearDown(self):
self.cache.close()
self.directory.cleanup()
def test_fresh_cache_and_callers_cannot_mutate_snapshot(self):
loader = MagicMock(return_value=copy.deepcopy(ROWS))
first = self.cache.read(loader)
first[1][2] = "changed outside cache"
self.assertEqual(self.cache.read(loader), ROWS)
loader.assert_called_once()
def test_stale_reads_are_immediate_and_refresh_is_single_flight(self):
self.cache.publish(ROWS)
self.cache.invalidate()
started, release = threading.Event(), threading.Event()
def slow_load():
started.set()
release.wait(2)
return [["new"]]
loader = MagicMock(side_effect=slow_load)
try:
began = time.monotonic()
self.assertEqual(self.cache.read(loader), ROWS)
self.assertLess(time.monotonic() - began, 0.2)
self.assertTrue(started.wait(1))
for _ in range(20):
self.assertEqual(self.cache.read(loader), ROWS)
self.assertEqual(loader.call_count, 1)
finally:
release.set()
self.cache._pending.result(2)
self.assertEqual(self.cache.read(loader), [["new"]])
def test_restart_restores_snapshot_when_google_is_down(self):
self.cache.publish(ROWS)
restored = runtime.SettingsCache(self.cache.path, "test")
try:
loader = MagicMock(side_effect=requests.ReadTimeout())
self.assertEqual(restored.read(loader), ROWS)
restored._pending.result(2)
for _ in range(10):
self.assertEqual(restored.read(loader), ROWS)
self.assertEqual(loader.call_count, 1)
finally:
restored.close()
def test_background_read_cannot_replace_a_later_save(self):
self.cache.publish(ROWS)
self.cache.invalidate()
started, release = threading.Event(), threading.Event()
def slow_load():
started.set()
release.wait(2)
return ROWS
try:
self.cache.read(slow_load)
self.assertTrue(started.wait(1))
self.cache.publish([["saved later"]])
finally:
release.set()
self.cache._pending.result(2)
self.assertEqual(self.cache.read(slow_load), [["saved later"]])
self.assertEqual(json.loads(self.cache.path.read_text(encoding="utf-8"))["rows"], [["saved later"]])
def test_force_refresh_failure_is_not_hidden_by_cache(self):
self.cache.publish(ROWS)
with self.assertRaises(requests.ReadTimeout):
self.cache.read(MagicMock(side_effect=requests.ReadTimeout()), force=True)
self.assertEqual(self.cache.read(MagicMock()), ROWS)
def test_cache_cannot_be_reused_for_a_different_spreadsheet(self):
self.cache.publish(ROWS)
other = runtime.SettingsCache(self.cache.path, "different-sheet")
try:
self.assertFalse(other.has_value)
finally:
other.close()
class TimeoutTests(unittest.TestCase):
def test_auth_refresh_default_timeout_is_capped(self):
with patch.object(requests.Session, "request", return_value=object()) as send:
runtime.GoogleTimeoutSession().request("POST", "https://example.invalid", timeout=120)
self.assertEqual(send.call_args.kwargs["timeout"], runtime.GOOGLE_TIMEOUT)
def test_authorized_requests_are_bounded_even_with_no_timeout(self):
session = runtime.create_google_session(MagicMock())
with patch.object(runtime.AuthorizedSession, "request", return_value=object()) as send:
session.request("GET", "https://example.invalid", timeout=None)
self.assertEqual(send.call_args.kwargs["timeout"], runtime.GOOGLE_TIMEOUT)
class ApiTests(unittest.IsolatedAsyncioTestCase):
def setUp(self):
# These tests exercise the legacy Google backend regardless of local .env.
backend = patch.object(main, "SETTINGS_BACKEND", "google")
backend.start()
self.addCleanup(backend.stop)
histories = patch.object(main, "HISTORY_BACKEND", "google")
histories.start()
self.addCleanup(histories.stop)
manual = patch.object(main, "MANUAL_ORDERS_BACKEND", "google")
manual.start()
self.addCleanup(manual.stop)
self.directory = tempfile.TemporaryDirectory()
self.cache = runtime.SettingsCache(Path(self.directory.name) / "settings.json", "test")
self.cache_patch = patch.object(main, "_settings_cache", self.cache)
self.cache_patch.start()
def tearDown(self):
self.cache_patch.stop()
self.cache.close()
self.directory.cleanup()
async def test_google_hang_does_not_block_page_or_db_records(self):
started, release = threading.Event(), threading.Event()
def stalled_google():
started.set()
release.wait(3)
raise requests.ReadTimeout()
connection = MagicMock()
connection.cursor.return_value.__enter__.return_value.fetchall.return_value = []
with patch.object(main, "_load_settings_rows", stalled_google), \
patch.object(records, "_ensure_schema"), patch.object(records, "_connect", return_value=connection):
config_task = asyncio.create_task(request("GET", "/api/config"))
try:
await wait_until(started.is_set)
async with asyncio.timeout(1):
page = await request("GET", "/")
db = await request("GET", "/api/cs-records?record_date=2026-10-19")
self.assertEqual(page[0], 200)
self.assertEqual(db[0], 200)
self.assertEqual(json.loads(db[2])["records"], [])
self.assertFalse(config_task.done())
finally:
release.set()
response = await config_task
self.assertEqual(response[0], 503)
async def test_saturation_keeps_db_pool_and_cached_settings_available(self):
release = threading.Event()
started = []
def stalled():
started.append(True)
release.wait(3)
self.cache.publish(ROWS)
tasks = [asyncio.create_task(runtime.run_google_io(stalled)) for _ in range(runtime.GOOGLE_WORKERS)]
try:
await wait_until(lambda: len(started) == runtime.GOOGLE_WORKERS)
with self.assertRaises(main.HTTPException) as error:
await runtime.run_google_io(lambda: None)
self.assertEqual(error.exception.status_code, 503)
async with asyncio.timeout(1):
config = await request("GET", "/api/config")
page = await request("GET", "/")
self.assertEqual(config[0], 200)
self.assertEqual(page[0], 200)
finally:
release.set()
await asyncio.gather(*tasks)
async def test_settings_save_uses_one_atomic_write_and_updates_cache(self):
self.cache.publish(ROWS)
worksheet = MagicMock(id=123, row_count=999, col_count=26)
worksheet.get_all_values.return_value = copy.deepcopy(ROWS)
with patch.object(main, "get_settings_worksheet", return_value=worksheet):
saved = await request("POST", "/api/app/settings", {"theme": "new"})
self.assertEqual(saved[0], 200)
worksheet.clear.assert_not_called()
worksheet.update.assert_not_called()
worksheet.spreadsheet.batch_update.assert_called_once()
body = worksheet.spreadsheet.batch_update.call_args.args[0]
change = body["requests"][-1]["updateCells"]
self.assertEqual(change["range"], {"sheetId": 123})
self.assertEqual(change["fields"], "userEnteredValue")
self.assertEqual(change["rows"][1]["values"][2]["userEnteredValue"]["stringValue"], "new")
response = await request("GET", "/api/config")
self.assertEqual(json.loads(response[2])["APP_SETTINGS"]["theme"], "new")
self.assertEqual(worksheet.get_all_values.call_count, 1)
async def test_save_does_not_use_stale_settings_during_outage(self):
self.cache.publish(ROWS)
worksheet = MagicMock()
worksheet.get_all_values.side_effect = requests.ReadTimeout()
with patch.object(main, "get_settings_worksheet", return_value=worksheet):
response = await request("POST", "/api/app/settings", {"theme": "new"})
self.assertEqual(response[0], 503)
worksheet.spreadsheet.batch_update.assert_not_called()
self.assertEqual(self.cache.read(MagicMock()), ROWS)
async def test_failed_write_never_publishes_unconfirmed_values(self):
self.cache.publish(ROWS)
worksheet = MagicMock(id=123, row_count=999, col_count=26)
worksheet.get_all_values.return_value = copy.deepcopy(ROWS)
worksheet.spreadsheet.batch_update.side_effect = requests.ReadTimeout()
with patch.object(main, "get_settings_worksheet", return_value=worksheet):
response = await request("POST", "/api/app/settings", {"theme": "new"})
self.assertEqual(response[0], 503)
self.assertEqual(self.cache._rows, ROWS)
self.assertTrue(self.cache.stale)
async def test_concurrent_settings_updates_preserve_both_changes(self):
remote_rows = copy.deepcopy(ROWS)
worksheet = MagicMock(id=123, row_count=999, col_count=26)
worksheet.get_all_values.side_effect = lambda: copy.deepcopy(remote_rows)
def atomic_save(body):
remote_rows[:] = [[c["userEnteredValue"]["stringValue"] for c in row["values"]]
for row in body["requests"][-1]["updateCells"]["rows"]]
worksheet.spreadsheet.batch_update.side_effect = atomic_save
with patch.object(main, "get_settings_worksheet", return_value=worksheet):
results = await asyncio.gather(
request("POST", "/api/app/settings", {"first": "1"}),
request("POST", "/api/app/settings", {"second": "2"}),
)
self.assertTrue(all(result[0] == 200 for result in results))
result = await request("GET", "/api/config")
settings = json.loads(result[2])["APP_SETTINGS"]
self.assertEqual(settings["first"], "1")
self.assertEqual(settings["second"], "2")
if __name__ == "__main__":
unittest.main()