king
279 lines
12 KiB
Python
279 lines
12 KiB
Python
from sqlalchemy import inspect, text
|
|
|
|
|
|
SEARCH_OPTIMIZATION_VERSION = "2026-04-24-search-v1"
|
|
PHONE_FTS_VERSION = "2026-04-24-phone-fts-v1"
|
|
ADDRESS_NORMALIZATION_VERSION = "2026-04-29-address-normalized-v1"
|
|
|
|
|
|
def _supports_trigram_fts(conn) -> bool:
|
|
try:
|
|
conn.execute(
|
|
text(
|
|
"CREATE VIRTUAL TABLE IF NOT EXISTS temp_fts_trigram_check "
|
|
"USING fts5(value, tokenize='trigram')"
|
|
)
|
|
)
|
|
conn.execute(text("DROP TABLE IF EXISTS temp_fts_trigram_check"))
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _setting_value(conn, key: str) -> str | None:
|
|
row = conn.execute(
|
|
text("SELECT value FROM settings WHERE key = :key"), {"key": key}
|
|
).fetchone()
|
|
return row[0] if row else None
|
|
|
|
|
|
def _set_setting(conn, key: str, value: str) -> None:
|
|
conn.execute(
|
|
text(
|
|
"INSERT INTO settings(key, value) VALUES (:key, :value) "
|
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
|
),
|
|
{"key": key, "value": value},
|
|
)
|
|
|
|
|
|
def ensure_search_optimizations(engine) -> None:
|
|
if engine.dialect.name != "sqlite":
|
|
ensure_standard_indexes(engine)
|
|
return
|
|
|
|
inspector = inspect(engine)
|
|
existing_columns = {column["name"] for column in inspector.get_columns("orders")}
|
|
|
|
with engine.begin() as conn:
|
|
needs_rebuild = _setting_value(conn, "search_optimization_version") != SEARCH_OPTIMIZATION_VERSION
|
|
if needs_rebuild:
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_ai"))
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_ad"))
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_au"))
|
|
conn.execute(text("DROP TABLE IF EXISTS orders_name_address_fts"))
|
|
|
|
if "recipient_phone_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_phone_normalized TEXT"))
|
|
if "recipient_mobile_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_mobile_normalized TEXT"))
|
|
if "tracking_number_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN tracking_number_normalized TEXT"))
|
|
if "address_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN address_normalized TEXT"))
|
|
added_customer_note = "customer_note" not in existing_columns
|
|
if added_customer_note:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN customer_note TEXT"))
|
|
if "order_note" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN order_note TEXT"))
|
|
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_phone_normalized "
|
|
"ON orders(recipient_phone_normalized)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_mobile_normalized "
|
|
"ON orders(recipient_mobile_normalized)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_tracking_normalized "
|
|
"ON orders(tracking_number_normalized)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_address_normalized "
|
|
"ON orders(address_normalized)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_customer_note "
|
|
"ON orders(customer_note)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_order_note "
|
|
"ON orders(order_note)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_tracking_number_id "
|
|
"ON orders(tracking_number, id DESC)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_upload_date_id "
|
|
"ON orders(upload_date, id DESC)"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE INDEX IF NOT EXISTS ix_orders_order_date_id "
|
|
"ON orders(order_date, id DESC)"
|
|
)
|
|
)
|
|
|
|
if needs_rebuild:
|
|
conn.execute(
|
|
text(
|
|
"UPDATE orders SET "
|
|
"recipient_phone_normalized = replace(replace(replace(replace(replace(coalesce(recipient_phone, ''), '-', ''), ' ', ''), '.', ''), ')', ''), '(', ''), "
|
|
"recipient_mobile_normalized = replace(replace(replace(replace(replace(coalesce(recipient_mobile, ''), '-', ''), ' ', ''), '.', ''), ')', ''), '(', ''), "
|
|
"tracking_number_normalized = upper(replace(replace(replace(replace(coalesce(tracking_number, ''), '-', ''), ' ', ''), '/', ''), '.', ''))"
|
|
)
|
|
)
|
|
if added_customer_note:
|
|
conn.execute(
|
|
text(
|
|
"UPDATE orders SET customer_note = note "
|
|
"WHERE note IS NOT NULL AND note != ''"
|
|
)
|
|
)
|
|
|
|
address_normalization_rebuild = (
|
|
_setting_value(conn, "address_normalization_version") != ADDRESS_NORMALIZATION_VERSION
|
|
)
|
|
if address_normalization_rebuild:
|
|
conn.execute(
|
|
text(
|
|
"UPDATE orders SET address_normalized = "
|
|
"trim(coalesce(address, ''), "
|
|
"' ' || char(9) || char(10) || char(13) || char(160) || "
|
|
"char(8203) || char(8204) || char(8205) || char(65279))"
|
|
)
|
|
)
|
|
_set_setting(conn, "address_normalization_version", ADDRESS_NORMALIZATION_VERSION)
|
|
|
|
tokenizer = "trigram" if _supports_trigram_fts(conn) else "unicode61"
|
|
conn.execute(
|
|
text(
|
|
"CREATE VIRTUAL TABLE IF NOT EXISTS orders_name_address_fts "
|
|
"USING fts5("
|
|
"recipient_name, address, "
|
|
"content='orders', content_rowid='id', "
|
|
f"tokenize='{tokenizer}'"
|
|
")"
|
|
)
|
|
)
|
|
|
|
if needs_rebuild:
|
|
conn.execute(text("INSERT INTO orders_name_address_fts(orders_name_address_fts) VALUES('rebuild')"))
|
|
_set_setting(conn, "search_optimization_version", SEARCH_OPTIMIZATION_VERSION)
|
|
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_fts_ai AFTER INSERT ON orders BEGIN "
|
|
"INSERT INTO orders_name_address_fts(rowid, recipient_name, address) "
|
|
"VALUES (new.id, new.recipient_name, new.address); "
|
|
"END"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_fts_ad AFTER DELETE ON orders BEGIN "
|
|
"INSERT INTO orders_name_address_fts(orders_name_address_fts, rowid, recipient_name, address) "
|
|
"VALUES('delete', old.id, old.recipient_name, old.address); "
|
|
"END"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_fts_au AFTER UPDATE ON orders BEGIN "
|
|
"INSERT INTO orders_name_address_fts(orders_name_address_fts, rowid, recipient_name, address) "
|
|
"VALUES('delete', old.id, old.recipient_name, old.address); "
|
|
"INSERT INTO orders_name_address_fts(rowid, recipient_name, address) "
|
|
"VALUES (new.id, new.recipient_name, new.address); "
|
|
"END"
|
|
)
|
|
)
|
|
|
|
phone_fts_rebuild = _setting_value(conn, "phone_fts_version") != PHONE_FTS_VERSION
|
|
if phone_fts_rebuild:
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_ai"))
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_ad"))
|
|
conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_au"))
|
|
conn.execute(text("DROP TABLE IF EXISTS orders_phone_fts"))
|
|
conn.execute(
|
|
text(
|
|
"CREATE VIRTUAL TABLE IF NOT EXISTS orders_phone_fts "
|
|
"USING fts5("
|
|
"recipient_phone_normalized, recipient_mobile_normalized, "
|
|
"content='orders', content_rowid='id', tokenize='trigram'"
|
|
")"
|
|
)
|
|
)
|
|
if phone_fts_rebuild:
|
|
conn.execute(text("INSERT INTO orders_phone_fts(orders_phone_fts) VALUES('rebuild')"))
|
|
_set_setting(conn, "phone_fts_version", PHONE_FTS_VERSION)
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_phone_fts_ai AFTER INSERT ON orders BEGIN "
|
|
"INSERT INTO orders_phone_fts(rowid, recipient_phone_normalized, recipient_mobile_normalized) "
|
|
"VALUES (new.id, new.recipient_phone_normalized, new.recipient_mobile_normalized); "
|
|
"END"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_phone_fts_ad AFTER DELETE ON orders BEGIN "
|
|
"INSERT INTO orders_phone_fts(orders_phone_fts, rowid, recipient_phone_normalized, recipient_mobile_normalized) "
|
|
"VALUES('delete', old.id, old.recipient_phone_normalized, old.recipient_mobile_normalized); "
|
|
"END"
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"CREATE TRIGGER IF NOT EXISTS orders_phone_fts_au AFTER UPDATE ON orders BEGIN "
|
|
"INSERT INTO orders_phone_fts(orders_phone_fts, rowid, recipient_phone_normalized, recipient_mobile_normalized) "
|
|
"VALUES('delete', old.id, old.recipient_phone_normalized, old.recipient_mobile_normalized); "
|
|
"INSERT INTO orders_phone_fts(rowid, recipient_phone_normalized, recipient_mobile_normalized) "
|
|
"VALUES (new.id, new.recipient_phone_normalized, new.recipient_mobile_normalized); "
|
|
"END"
|
|
)
|
|
)
|
|
|
|
|
|
def ensure_standard_indexes(engine) -> None:
|
|
inspector = inspect(engine)
|
|
existing_columns = {column["name"] for column in inspector.get_columns("orders")}
|
|
|
|
with engine.begin() as conn:
|
|
if "recipient_phone_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_phone_normalized VARCHAR"))
|
|
if "recipient_mobile_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_mobile_normalized VARCHAR"))
|
|
if "tracking_number_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN tracking_number_normalized VARCHAR"))
|
|
if "address_normalized" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN address_normalized VARCHAR"))
|
|
added_customer_note = "customer_note" not in existing_columns
|
|
if added_customer_note:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN customer_note VARCHAR"))
|
|
if "order_note" not in existing_columns:
|
|
conn.execute(text("ALTER TABLE orders ADD COLUMN order_note VARCHAR"))
|
|
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_phone_normalized ON orders(recipient_phone_normalized)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_mobile_normalized ON orders(recipient_mobile_normalized)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_tracking_normalized ON orders(tracking_number_normalized)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_address_normalized ON orders(address_normalized)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_customer_note ON orders(customer_note)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_order_note ON orders(order_note)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_order_date_id ON orders(order_date, id DESC)"))
|
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_upload_date_id ON orders(upload_date, id DESC)"))
|
|
if added_customer_note:
|
|
conn.execute(
|
|
text(
|
|
"UPDATE orders SET customer_note = note "
|
|
"WHERE note IS NOT NULL AND note != ''"
|
|
)
|
|
)
|