Boostrap full-text search for the outbox

alembic/versions/2022_11_02_1914-368f511ad954_outbox_fts.py

@@ -0,0 +1,115 @@
+"""Outbox FTS
+
+Revision ID: 368f511ad954
+Revises: b28c0551c236
+Create Date: 2022-11-02 19:14:37.865923+00:00
+
+"""
+from sqlalchemy import insert
+from sqlalchemy import select
+from sqlalchemy.orm.session import Session
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '368f511ad954'
+down_revision = 'b28c0551c236'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # ### end Alembic commands ###
+    op.execute(
+        "CREATE VIRTUAL TABLE outbox_fts USING "
+        "fts5(summary, name, source, content='');"
+    )
+    op.execute(
+        "CREATE TRIGGER outbox_fts_ai AFTER "
+        "INSERT ON outbox WHEN new.ap_type in ('Article', 'Note', 'Question') BEGIN"
+        "    INSERT INTO outbox_fts (rowid, source, name, summary)"
+        "    VALUES ("
+        "        new.id, "
+        "        new.source, "
+        '        json_extract(new.ap_object, "$.name"), '
+        '        json_extract(new.ap_object, "$.summary")'
+        "     ); "
+        "END;"
+    )
+    op.execute(
+        "CREATE TRIGGER outbox_fts_ad AFTER "
+        "DELETE ON outbox WHEN old.ap_type in ('Article', 'Note', 'Question') BEGIN"
+        "    INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)"
+        "    VALUES ("
+        "        'delete', "
+        "        old.id, "
+        "        old.source, "
+        '        json_extract(old.ap_object, "$.name"), '
+        '        json_extract(old.ap_object, "$.summary")'
+        "    ); "
+        "END;"
+    )
+    op.execute(
+        "CREATE TRIGGER outbox_fts_au_softdelete AFTER "
+        "UPDATE ON outbox WHEN new.is_deleted = 1 AND "
+        "new.ap_type in ('Article', 'Note', 'Question') BEGIN"
+        "    INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)"
+        "    VALUES ("
+        "        'delete', "
+        "        old.id, "
+        "        old.source, "
+        '        json_extract(old.ap_object, "$.name"), '
+        '        json_extract(old.ap_object, "$.summary")'
+        "    ); "
+        "END; "
+    )
+    op.execute(
+        "CREATE TRIGGER outbox_fts_au AFTER "
+        "UPDATE ON outbox "
+        "WHEN (new.source <> old.source OR new.ap_object <> old.ap_object) AND "
+        "new.ap_type in ('Note', 'Article', 'Quesion') BEGIN"
+        "    INSERT INTO outbox_fts (outbox_fts, rowid, source, name, summary)"
+        "    VALUES ("
+        "        'delete', "
+        "        old.id, "
+        "        old.source, "
+        '        json_extract(old.ap_object, "$.name"), '
+        '        json_extract(old.ap_object, "$.summary")'
+        "    );"
+        "    INSERT INTO outbox_fts (rowid, source, name, summary)"
+        "    VALUES ("
+        "        new.id, "
+        "        new.source, "
+        '        json_extract(new.ap_object, "$.name"), '
+        '        json_extract(new.ap_object, "$.summary")'
+        "     );"
+        "END;"
+    )
+    from app.models import OutboxObject
+    from app.models import outbox_fts
+    sess = Session(op.get_bind())
+
+    # Backfill the index
+    outbox_objects = sess.execute(select(OutboxObject).where(
+        OutboxObject.ap_type.in_(["Article", "Note", "Question"]))
+    ).scalars()
+    for outbox_object in outbox_objects:
+        row = {"source": outbox_object.source, "rowid": outbox_object.id}
+        if name := outbox_object.ap_object.get("name"):
+            row["name"] = name
+        if summary := outbox_object.ap_object.get("summary"):
+            row["summary"] = summary
+        sess.execute(insert(outbox_fts).values(row))
+
+    sess.commit()
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # ### end Alembic commands ###
+    op.drop_table('outbox_fts')
+    op.execute("DROP TRIGGER outbox_fts_ai;")
+    op.execute("DROP TRIGGER outbox_fts_ad;")
+    op.execute("DROP TRIGGER outbox_fts_au_softdelete;")
+    op.execute("DROP TRIGGER outbox_fts_au;")

app/admin.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Any
 
 import httpx
 from fastapi import APIRouter
@@ -149,6 +150,57 @@ async def get_lookup(
     )
 
 
+@router.get("/search")
+async def admin_search(
+    request: Request,
+    query: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> templates.TemplateResponse | RedirectResponse:
+
+    results: list[Any] = []
+    if query:
+        results = (
+            (
+                await db_session.execute(
+                    select(models.OutboxObject)
+                    .join(
+                        models.outbox_fts,
+                        models.outbox_fts.c.rowid == models.OutboxObject.id,
+                    )
+                    .options(
+                        joinedload(
+                            models.OutboxObject.outbox_object_attachments
+                        ).options(joinedload(models.OutboxObjectAttachment.upload)),
+                        joinedload(models.OutboxObject.relates_to_inbox_object).options(
+                            joinedload(models.InboxObject.actor),
+                        ),
+                        joinedload(
+                            models.OutboxObject.relates_to_outbox_object
+                        ).options(
+                            joinedload(
+                                models.OutboxObject.outbox_object_attachments
+                            ).options(joinedload(models.OutboxObjectAttachment.upload)),
+                        ),
+                    )
+                    .where(models.outbox_fts.c.outbox_fts.op("MATCH")(query))
+                    .limit(20)
+                )
+            )  # type: ignore
+            .unique()
+            .scalars()
+        )
+
+    return await templates.render_template(
+        db_session,
+        request,
+        "admin_search.html",
+        {
+            "query": query,
+            "results": results,
+        },
+    )
+
+
 @router.get("/new")
 async def admin_new(
     request: Request,

app/templates/layout.html

@@ -30,6 +30,7 @@
         <li>{{ admin_link("admin_inbox", "Inbox") }} / {{ admin_link("admin_outbox", "Outbox") }}</li>
         <li>{{ admin_link("admin_direct_messages", "DMs") }}</li>
         <li>{{ admin_link("get_notifications", "Notifications") }} {% if notifications_count %}({{ notifications_count }}){% endif %}</li>
+        <li>{{ admin_link("admin_search", "Search") }}</li>
         <li>{{ admin_link("get_lookup", "Lookup") }}</li>
         <li>{{ admin_link("admin_bookmarks", "Bookmarks") }}</li>
         <li><a href="{{ url_for("logout")}}">Logout</a></li>