Start to merge IndieWeb and AP interactions

alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py

@@ -0,0 +1,32 @@
+"""Add Webmention.webmention_type
+
+Revision ID: fadfd359ce78
+Revises: b28c0551c236
+Create Date: 2022-11-16 19:42:56.925512+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'fadfd359ce78'
+down_revision = 'b28c0551c236'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('webmention_type', sa.Enum('UNKNOWN', 'LIKE', 'REPLY', 'REPOST', name='webmentiontype'), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.drop_column('webmention_type')
+
+    # ### end Alembic commands ###

app/boxes.py

@@ -201,7 +201,7 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None:
         raise ValueError("Should never happen")
 
     outbox_object_to_delete.is_deleted = True
-    await db_session.commit()
+    await db_session.flush()
 
     # Compute the original recipients
     recipients = await _compute_recipients(
@@ -216,14 +216,17 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None:
             db_session, outbox_object_to_delete.in_reply_to
         )
         if replied_object:
+            if replied_object.is_from_outbox:
+                # Different helper here because we also count webmentions
+                new_replies_count = await _get_outbox_replies_count(
+                    db_session, replied_object  # type: ignore
+                )
+            else:
                 new_replies_count = await _get_replies_count(
                     db_session, replied_object.ap_id
                 )
 
             replied_object.replies_count = new_replies_count
-            if replied_object.replies_count < 0:
-                logger.warning("negative replies count for {replied_object.ap_id}")
-                replied_object.replies_count = 0
         else:
             logger.info(f"{outbox_object_to_delete.in_reply_to} not found")
 
@@ -1048,6 +1051,32 @@ async def get_outbox_object_by_ap_id(
     )  # type: ignore
 
 
+async def get_outbox_object_by_slug_and_short_id(
+    db_session: AsyncSession,
+    slug: str,
+    short_id: str,
+) -> models.OutboxObject | None:
+    return (
+        (
+            await db_session.execute(
+                select(models.OutboxObject)
+                .options(
+                    joinedload(models.OutboxObject.outbox_object_attachments).options(
+                        joinedload(models.OutboxObjectAttachment.upload)
+                    )
+                )
+                .where(
+                    models.OutboxObject.public_id.like(f"{short_id}%"),
+                    models.OutboxObject.slug == slug,
+                    models.OutboxObject.is_deleted.is_(False),
+                )
+            )
+        )
+        .unique()
+        .scalar_one_or_none()
+    )
+
+
 async def get_anybox_object_by_ap_id(
     db_session: AsyncSession, ap_id: str
 ) -> AnyboxObject | None:
@@ -1201,6 +1230,67 @@ async def _get_replies_count(
     )
 
 
+async def _get_outbox_replies_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (await _get_replies_count(db_session, outbox_object.ap_id)) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.REPLY,
+            )
+        )
+    )
+
+
+async def _get_outbox_likes_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (
+        await db_session.scalar(
+            select(func.count(models.InboxObject.id)).where(
+                models.InboxObject.ap_type == "Like",
+                models.InboxObject.relates_to_outbox_object_id == outbox_object.id,
+                models.InboxObject.is_deleted.is_(False),
+            )
+        )
+    ) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.LIKE,
+            )
+        )
+    )
+
+
+async def _get_outbox_announces_count(
+    db_session: AsyncSession,
+    outbox_object: models.OutboxObject,
+) -> int:
+    return (
+        await db_session.scalar(
+            select(func.count(models.InboxObject.id)).where(
+                models.InboxObject.ap_type == "Announce",
+                models.InboxObject.relates_to_outbox_object_id == outbox_object.id,
+                models.InboxObject.is_deleted.is_(False),
+            )
+        )
+    ) + (
+        await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == outbox_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.REPOST,
+            )
+        )
+    )
+
+
 async def _revert_side_effect_for_deleted_object(
     db_session: AsyncSession,
     delete_activity: models.InboxObject | None,
@@ -1231,8 +1321,8 @@ async def _revert_side_effect_for_deleted_object(
                 # also needs to be forwarded
                 is_delete_needs_to_be_forwarded = True
 
-                new_replies_count = await _get_replies_count(
+                new_replies_count = await _get_outbox_replies_count(
-                    db_session, replied_object.ap_id
+                    db_session, replied_object  # type: ignore
                 )
 
                 await db_session.execute(
@@ -1262,12 +1352,13 @@ async def _revert_side_effect_for_deleted_object(
         )
         if related_object:
             if related_object.is_from_outbox:
+                likes_count = await _get_outbox_likes_count(db_session, related_object)
                 await db_session.execute(
                     update(models.OutboxObject)
                     .where(
                         models.OutboxObject.id == related_object.id,
                     )
-                    .values(likes_count=models.OutboxObject.likes_count - 1)
+                    .values(likes_count=likes_count - 1)
                 )
     elif (
         deleted_ap_object.ap_type == "Annouce"
@@ -1279,12 +1370,15 @@ async def _revert_side_effect_for_deleted_object(
         )
         if related_object:
             if related_object.is_from_outbox:
+                announces_count = await _get_outbox_announces_count(
+                    db_session, related_object
+                )
                 await db_session.execute(
                     update(models.OutboxObject)
                     .where(
                         models.OutboxObject.id == related_object.id,
                     )
-                    .values(announces_count=models.OutboxObject.announces_count - 1)
+                    .values(announces_count=announces_count - 1)
                 )
 
     # Delete any Like/Announce
@@ -1826,8 +1920,8 @@ async def _process_note_object(
                         replied_object,  # type: ignore  # outbox check below
                     )
                 else:
-                    new_replies_count = await _get_replies_count(
+                    new_replies_count = await _get_outbox_replies_count(
-                        db_session, replied_object.ap_id
+                        db_session, replied_object  # type: ignore
                     )
 
                     await db_session.execute(
@@ -2073,7 +2167,10 @@ async def _handle_like_activity(
         )
         await db_session.delete(like_activity)
     else:
-        relates_to_outbox_object.likes_count = models.OutboxObject.likes_count + 1
+        relates_to_outbox_object.likes_count = await _get_outbox_likes_count(
+            db_session,
+            relates_to_outbox_object,
+        )
 
         notif = models.Notification(
             notification_type=models.NotificationType.LIKE,

app/main.py

@@ -799,24 +799,8 @@ async def article_by_slug(
     db_session: AsyncSession = Depends(get_db_session),
     httpsig_info: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker),
 ) -> ActivityPubResponse | templates.TemplateResponse | RedirectResponse:
-    maybe_object = (
+    maybe_object = await boxes.get_outbox_object_by_slug_and_short_id(
-        (
+        db_session, slug, short_id
-            await db_session.execute(
-                select(models.OutboxObject)
-                .options(
-                    joinedload(models.OutboxObject.outbox_object_attachments).options(
-                        joinedload(models.OutboxObjectAttachment.upload)
-                    )
-                )
-                .where(
-                    models.OutboxObject.public_id.like(f"{short_id}%"),
-                    models.OutboxObject.slug == slug,
-                    models.OutboxObject.is_deleted.is_(False),
-                )
-            )
-        )
-        .unique()
-        .scalar_one_or_none()
     )
     if not maybe_object:
         raise HTTPException(status_code=404)

app/models.py

@@ -468,6 +468,14 @@ class IndieAuthAccessToken(Base):
     is_revoked = Column(Boolean, nullable=False, default=False)
 
 
+@enum.unique
+class WebmentionType(str, enum.Enum):
+    UNKNOWN = "unknown"
+    LIKE = "like"
+    REPLY = "reply"
+    REPOST = "repost"
+
+
 class Webmention(Base):
     __tablename__ = "webmention"
     __table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),)
@@ -484,6 +492,8 @@ class Webmention(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
     outbox_object = relationship(OutboxObject, uselist=False)
 
+    webmention_type = Column(Enum(WebmentionType), nullable=True)
+
     @property
     def as_facepile_item(self) -> webmentions.Webmention | None:
         if not self.source_microformats:
@@ -493,6 +503,7 @@ class Webmention(Base):
                 self.source_microformats["items"], self.source
             )
         except Exception:
+            # TODO: return a facepile with the unknown image
             logger.warning(
                 f"Failed to generate facefile item for Webmention id={self.id}"
             )

app/webmentions.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
@@ -9,7 +11,11 @@ from loguru import logger
 from sqlalchemy import select
 
 from app import models
+from app.boxes import _get_outbox_announces_count
+from app.boxes import _get_outbox_likes_count
+from app.boxes import _get_outbox_replies_count
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import get_outbox_object_by_slug_and_short_id
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.utils import microformats
@@ -47,6 +53,7 @@ async def webmention_endpoint(
 
         check_url(source)
         check_url(target)
+        parsed_target_url = urlparse(target)
     except Exception:
         logger.exception("Invalid webmention request")
         raise HTTPException(status_code=400, detail="Invalid payload")
@@ -65,6 +72,16 @@ async def webmention_endpoint(
         logger.info("Found existing Webmention, will try to update or delete")
 
     mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
+
+    if not mentioned_object and parsed_target_url.path.startswith("/articles/"):
+        try:
+            _, _, short_id, slug = parsed_target_url.path.split("/")
+            mentioned_object = await get_outbox_object_by_slug_and_short_id(
+                db_session, slug, short_id
+            )
+        except Exception:
+            logger.exception(f"Failed to match {target}")
+
     if not mentioned_object:
         logger.info(f"Invalid target {target=}")
 
@@ -90,8 +107,13 @@ async def webmention_endpoint(
         logger.warning(f"target {target=} not found in source")
         if existing_webmention_in_db:
             logger.info("Deleting existing Webmention")
-            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
             existing_webmention_in_db.is_deleted = True
+            await db_session.flush()
+
+            # Revert side effects
+            await _handle_webmention_side_effects(
+                db_session, existing_webmention_in_db, mentioned_object
+            )
 
             notif = models.Notification(
                 notification_type=models.NotificationType.DELETED_WEBMENTION,
@@ -110,10 +132,25 @@ async def webmention_endpoint(
         else:
             return JSONResponse(content={}, status_code=200)
 
+    webmention_type = models.WebmentionType.UNKNOWN
+    for item in data.get("items", []):
+        if target in item.get("properties", {}).get("in-reply-to", []):
+            webmention_type = models.WebmentionType.REPLY
+            break
+        elif target in item.get("properties", {}).get("like-of", []):
+            webmention_type = models.WebmentionType.LIKE
+            break
+        elif target in item.get("properties", {}).get("repost-of", []):
+            webmention_type = models.WebmentionType.REPOST
+            break
+
+    webmention: models.Webmention
     if existing_webmention_in_db:
         # Undelete if needed
         existing_webmention_in_db.is_deleted = False
         existing_webmention_in_db.source_microformats = data
+        await db_session.flush()
+        webmention = existing_webmention_in_db
 
         notif = models.Notification(
             notification_type=models.NotificationType.UPDATED_WEBMENTION,
@@ -127,9 +164,11 @@ async def webmention_endpoint(
             target=target,
             source_microformats=data,
             outbox_object_id=mentioned_object.id,
+            webmention_type=webmention_type,
         )
         db_session.add(new_webmention)
         await db_session.flush()
+        webmention = new_webmention
 
         notif = models.Notification(
             notification_type=models.NotificationType.NEW_WEBMENTION,
@@ -138,8 +177,32 @@ async def webmention_endpoint(
         )
         db_session.add(notif)
 
-        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
+    # Handle side effect
-
+    await _handle_webmention_side_effects(db_session, webmention, mentioned_object)
     await db_session.commit()
 
     return JSONResponse(content={}, status_code=200)
+
+
+async def _handle_webmention_side_effects(
+    db_session: AsyncSession,
+    webmention: models.Webmention,
+    mentioned_object: models.OutboxObject,
+) -> None:
+    if webmention.webmention_type == models.WebmentionType.UNKNOWN:
+        # TODO: recount everything
+        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
+    elif webmention.webmention_type == models.WebmentionType.LIKE:
+        mentioned_object.likes_count = await _get_outbox_likes_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPOST:
+        mentioned_object.announces_count = await _get_outbox_announces_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPLY:
+        mentioned_object.replies_count = await _get_outbox_replies_count(
+            db_session, mentioned_object
+        )
+    else:
+        raise ValueError(f"Unhandled {webmention.webmention_type} webmention")