from sqlalchemy import inspect, text SEARCH_OPTIMIZATION_VERSION = "2026-04-24-search-v1" PHONE_FTS_VERSION = "2026-04-24-phone-fts-v1" ADDRESS_NORMALIZATION_VERSION = "2026-04-29-address-normalized-v1" def _supports_trigram_fts(conn) -> bool: try: conn.execute( text( "CREATE VIRTUAL TABLE IF NOT EXISTS temp_fts_trigram_check " "USING fts5(value, tokenize='trigram')" ) ) conn.execute(text("DROP TABLE IF EXISTS temp_fts_trigram_check")) return True except Exception: return False def _setting_value(conn, key: str) -> str | None: row = conn.execute( text("SELECT value FROM settings WHERE key = :key"), {"key": key} ).fetchone() return row[0] if row else None def _set_setting(conn, key: str, value: str) -> None: conn.execute( text( "INSERT INTO settings(key, value) VALUES (:key, :value) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value" ), {"key": key, "value": value}, ) def ensure_search_optimizations(engine) -> None: if engine.dialect.name != "sqlite": ensure_standard_indexes(engine) return inspector = inspect(engine) existing_columns = {column["name"] for column in inspector.get_columns("orders")} with engine.begin() as conn: needs_rebuild = _setting_value(conn, "search_optimization_version") != SEARCH_OPTIMIZATION_VERSION if needs_rebuild: conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_ai")) conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_ad")) conn.execute(text("DROP TRIGGER IF EXISTS orders_fts_au")) conn.execute(text("DROP TABLE IF EXISTS orders_name_address_fts")) if "recipient_phone_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_phone_normalized TEXT")) if "recipient_mobile_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_mobile_normalized TEXT")) if "tracking_number_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN tracking_number_normalized TEXT")) if "address_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN address_normalized TEXT")) added_customer_note = "customer_note" not in existing_columns if added_customer_note: conn.execute(text("ALTER TABLE orders ADD COLUMN customer_note TEXT")) if "order_note" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN order_note TEXT")) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_phone_normalized " "ON orders(recipient_phone_normalized)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_mobile_normalized " "ON orders(recipient_mobile_normalized)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_tracking_normalized " "ON orders(tracking_number_normalized)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_address_normalized " "ON orders(address_normalized)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_customer_note " "ON orders(customer_note)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_order_note " "ON orders(order_note)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_tracking_number_id " "ON orders(tracking_number, id DESC)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_upload_date_id " "ON orders(upload_date, id DESC)" ) ) conn.execute( text( "CREATE INDEX IF NOT EXISTS ix_orders_order_date_id " "ON orders(order_date, id DESC)" ) ) if needs_rebuild: conn.execute( text( "UPDATE orders SET " "recipient_phone_normalized = replace(replace(replace(replace(replace(coalesce(recipient_phone, ''), '-', ''), ' ', ''), '.', ''), ')', ''), '(', ''), " "recipient_mobile_normalized = replace(replace(replace(replace(replace(coalesce(recipient_mobile, ''), '-', ''), ' ', ''), '.', ''), ')', ''), '(', ''), " "tracking_number_normalized = upper(replace(replace(replace(replace(coalesce(tracking_number, ''), '-', ''), ' ', ''), '/', ''), '.', ''))" ) ) if added_customer_note: conn.execute( text( "UPDATE orders SET customer_note = note " "WHERE note IS NOT NULL AND note != ''" ) ) address_normalization_rebuild = ( _setting_value(conn, "address_normalization_version") != ADDRESS_NORMALIZATION_VERSION ) if address_normalization_rebuild: conn.execute( text( "UPDATE orders SET address_normalized = " "trim(coalesce(address, ''), " "' ' || char(9) || char(10) || char(13) || char(160) || " "char(8203) || char(8204) || char(8205) || char(65279))" ) ) _set_setting(conn, "address_normalization_version", ADDRESS_NORMALIZATION_VERSION) tokenizer = "trigram" if _supports_trigram_fts(conn) else "unicode61" conn.execute( text( "CREATE VIRTUAL TABLE IF NOT EXISTS orders_name_address_fts " "USING fts5(" "recipient_name, address, " "content='orders', content_rowid='id', " f"tokenize='{tokenizer}'" ")" ) ) if needs_rebuild: conn.execute(text("INSERT INTO orders_name_address_fts(orders_name_address_fts) VALUES('rebuild')")) _set_setting(conn, "search_optimization_version", SEARCH_OPTIMIZATION_VERSION) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_fts_ai AFTER INSERT ON orders BEGIN " "INSERT INTO orders_name_address_fts(rowid, recipient_name, address) " "VALUES (new.id, new.recipient_name, new.address); " "END" ) ) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_fts_ad AFTER DELETE ON orders BEGIN " "INSERT INTO orders_name_address_fts(orders_name_address_fts, rowid, recipient_name, address) " "VALUES('delete', old.id, old.recipient_name, old.address); " "END" ) ) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_fts_au AFTER UPDATE ON orders BEGIN " "INSERT INTO orders_name_address_fts(orders_name_address_fts, rowid, recipient_name, address) " "VALUES('delete', old.id, old.recipient_name, old.address); " "INSERT INTO orders_name_address_fts(rowid, recipient_name, address) " "VALUES (new.id, new.recipient_name, new.address); " "END" ) ) phone_fts_rebuild = _setting_value(conn, "phone_fts_version") != PHONE_FTS_VERSION if phone_fts_rebuild: conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_ai")) conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_ad")) conn.execute(text("DROP TRIGGER IF EXISTS orders_phone_fts_au")) conn.execute(text("DROP TABLE IF EXISTS orders_phone_fts")) conn.execute( text( "CREATE VIRTUAL TABLE IF NOT EXISTS orders_phone_fts " "USING fts5(" "recipient_phone_normalized, recipient_mobile_normalized, " "content='orders', content_rowid='id', tokenize='trigram'" ")" ) ) if phone_fts_rebuild: conn.execute(text("INSERT INTO orders_phone_fts(orders_phone_fts) VALUES('rebuild')")) _set_setting(conn, "phone_fts_version", PHONE_FTS_VERSION) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_phone_fts_ai AFTER INSERT ON orders BEGIN " "INSERT INTO orders_phone_fts(rowid, recipient_phone_normalized, recipient_mobile_normalized) " "VALUES (new.id, new.recipient_phone_normalized, new.recipient_mobile_normalized); " "END" ) ) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_phone_fts_ad AFTER DELETE ON orders BEGIN " "INSERT INTO orders_phone_fts(orders_phone_fts, rowid, recipient_phone_normalized, recipient_mobile_normalized) " "VALUES('delete', old.id, old.recipient_phone_normalized, old.recipient_mobile_normalized); " "END" ) ) conn.execute( text( "CREATE TRIGGER IF NOT EXISTS orders_phone_fts_au AFTER UPDATE ON orders BEGIN " "INSERT INTO orders_phone_fts(orders_phone_fts, rowid, recipient_phone_normalized, recipient_mobile_normalized) " "VALUES('delete', old.id, old.recipient_phone_normalized, old.recipient_mobile_normalized); " "INSERT INTO orders_phone_fts(rowid, recipient_phone_normalized, recipient_mobile_normalized) " "VALUES (new.id, new.recipient_phone_normalized, new.recipient_mobile_normalized); " "END" ) ) def ensure_standard_indexes(engine) -> None: inspector = inspect(engine) existing_columns = {column["name"] for column in inspector.get_columns("orders")} with engine.begin() as conn: if "recipient_phone_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_phone_normalized VARCHAR")) if "recipient_mobile_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN recipient_mobile_normalized VARCHAR")) if "tracking_number_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN tracking_number_normalized VARCHAR")) if "address_normalized" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN address_normalized VARCHAR")) added_customer_note = "customer_note" not in existing_columns if added_customer_note: conn.execute(text("ALTER TABLE orders ADD COLUMN customer_note VARCHAR")) if "order_note" not in existing_columns: conn.execute(text("ALTER TABLE orders ADD COLUMN order_note VARCHAR")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_phone_normalized ON orders(recipient_phone_normalized)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_mobile_normalized ON orders(recipient_mobile_normalized)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_tracking_normalized ON orders(tracking_number_normalized)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_address_normalized ON orders(address_normalized)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_customer_note ON orders(customer_note)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_order_note ON orders(order_note)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_order_date_id ON orders(order_date, id DESC)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_upload_date_id ON orders(upload_date, id DESC)")) if added_customer_note: conn.execute( text( "UPDATE orders SET customer_note = note " "WHERE note IS NOT NULL AND note != ''" ) )