Cleanup and improved webmentions support

alembic/versions/69ce9fbdc483_add_webmentions_count.py

@@ -0,0 +1,28 @@
+"""Add webmentions count
+
+Revision ID: 69ce9fbdc483
+Revises: 1647cef23e9b
+Create Date: 2022-07-14 15:35:01.716133
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '69ce9fbdc483'
+down_revision = '1647cef23e9b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('outbox', sa.Column('webmentions_count', sa.Integer(), server_default='0', nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('outbox', 'webmentions_count')
+    # ### end Alembic commands ###

alembic/versions/fd23d95e5c16_improved_webmentions.py

@@ -0,0 +1,48 @@
+"""Improved Webmentions
+
+Revision ID: fd23d95e5c16
+Revises: 69ce9fbdc483
+Create Date: 2022-07-14 16:10:54.202455
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'fd23d95e5c16'
+down_revision = '69ce9fbdc483'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('webmention',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('is_deleted', sa.Boolean(), nullable=False),
+    sa.Column('source', sa.String(), nullable=False),
+    sa.Column('source_microformats', sa.JSON(), nullable=True),
+    sa.Column('target', sa.String(), nullable=False),
+    sa.Column('outbox_object_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['outbox_object_id'], ['outbox.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('source', 'target', name='uix_source_target')
+    )
+    op.create_index(op.f('ix_webmention_id'), 'webmention', ['id'], unique=False)
+    op.create_index(op.f('ix_webmention_source'), 'webmention', ['source'], unique=True)
+    op.create_index(op.f('ix_webmention_target'), 'webmention', ['target'], unique=False)
+    op.drop_column('outbox', 'webmentions')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('outbox', sa.Column('webmentions', sqlite.JSON(), nullable=True))
+    op.drop_index(op.f('ix_webmention_target'), table_name='webmention')
+    op.drop_index(op.f('ix_webmention_source'), table_name='webmention')
+    op.drop_index(op.f('ix_webmention_id'), table_name='webmention')
+    op.drop_table('webmention')
+    # ### end Alembic commands ###

app/boxes.py

@@ -27,12 +27,12 @@ from app.ap_object import RemoteObject
 from app.config import BASE_URL
 from app.config import ID
 from app.database import AsyncSession
-from app.database import now
 from app.outgoing_activities import new_outgoing_activity
 from app.source import markdownify
 from app.uploads import upload_to_attachment
 from app.utils import opengraph
 from app.utils import webmentions
+from app.utils.datetime import now
 from app.utils.datetime import parse_isoformat
 
 AnyboxObject = models.InboxObject | models.OutboxObject

app/database.py

@@ -1,4 +1,3 @@
-import datetime
 from typing import Any
 from typing import AsyncGenerator
 
@@ -23,10 +22,6 @@ async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit
 Base: Any = declarative_base()
 
 
-def now() -> datetime.datetime:
-    return datetime.datetime.now(datetime.timezone.utc)
-
-
 async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
     async with async_session() as session:
         try:

app/incoming_activities.py

@@ -13,7 +13,7 @@ from app import models
 from app.boxes import save_to_inbox
 from app.database import AsyncSession
 from app.database import async_session
-from app.database import now
+from app.utils.datetime import now
 
 _MAX_RETRIES = 5
 
@@ -63,7 +63,7 @@ async def process_next_incoming_activity(db_session: AsyncSession) -> bool:
         select(func.count(models.IncomingActivity.id)).where(*where)
     )
     if q_count > 0:
-        logger.info(f"{q_count} outgoing activities ready to process")
+        logger.info(f"{q_count} incoming activities ready to process")
     if not q_count:
         # logger.debug("No activities to process")
         return False

app/indieauth.py

@@ -21,8 +21,8 @@ from app.admin import user_session_or_redirect
 from app.config import verify_csrf_token
 from app.database import AsyncSession
 from app.database import get_db_session
-from app.database import now
 from app.utils import indieauth
+from app.utils.datetime import now
 
 router = APIRouter()
 

app/main.py

@@ -551,6 +551,17 @@ async def outbox_by_public_id(
         .all()
     )
 
+    webmentions = (
+        await db_session.scalars(
+            select(models.Webmention)
+            .filter(
+                models.Webmention.outbox_object_id == maybe_object.id,
+                models.Webmention.is_deleted.is_(False),
+            )
+            .limit(10)
+        )
+    ).all()
+
     return await templates.render_template(
         db_session,
         request,
@@ -560,6 +571,7 @@ async def outbox_by_public_id(
             "outbox_object": maybe_object,
             "likes": likes,
             "shares": shares,
+            "webmentions": webmentions,
         },
     )
 

app/models.py

@@ -3,6 +3,7 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
+from loguru import logger
 from sqlalchemy import JSON
 from sqlalchemy import Boolean
 from sqlalchemy import Column
@@ -22,7 +23,8 @@ from app.ap_object import Attachment
 from app.ap_object import Object as BaseObject
 from app.config import BASE_URL
 from app.database import Base
-from app.database import now
+from app.utils import webmentions
+from app.utils.datetime import now
 
 
 class Actor(Base, BaseActor):
@@ -152,10 +154,11 @@ class OutboxObject(Base, BaseObject):
     likes_count = Column(Integer, nullable=False, default=0)
     announces_count = Column(Integer, nullable=False, default=0)
     replies_count = Column(Integer, nullable=False, default=0)
+    webmentions_count: Mapped[int] = Column(
+        Integer, nullable=False, default=0, server_default="0"
+    )
     # reactions: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 
-    webmentions = Column(JSON, nullable=True)
-
     og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 
     # For the featured collection
@@ -457,3 +460,34 @@ class IndieAuthAccessToken(Base):
     expires_in = Column(Integer, nullable=False)
     scope = Column(String, nullable=False)
     is_revoked = Column(Boolean, nullable=False, default=False)
+
+
+class Webmention(Base):
+    __tablename__ = "webmention"
+    __table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),)
+
+    id = Column(Integer, primary_key=True, index=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+
+    is_deleted = Column(Boolean, nullable=False, default=False)
+
+    source: Mapped[str] = Column(String, nullable=False, index=True, unique=True)
+    source_microformats: Mapped[dict[str, Any] | None] = Column(JSON, nullable=True)
+
+    target = Column(String, nullable=False, index=True)
+    outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
+    outbox_object = relationship(OutboxObject, uselist=False)
+
+    @property
+    def as_facepile_item(self) -> webmentions.Webmention | None:
+        if not self.source_microformats:
+            return None
+        try:
+            return webmentions.Webmention.from_microformats(
+                self.source_microformats["items"], self.source
+            )
+        except Exception:
+            logger.warning(
+                f"Failed to generate facefile item for Webmention id={self.id}"
+            )
+            return None

app/outgoing_activities.py

@@ -20,8 +20,8 @@ from app.actor import _actor_hash
 from app.config import KEY_PATH
 from app.database import AsyncSession
 from app.database import SessionLocal
-from app.database import now
 from app.key import Key
+from app.utils.datetime import now
 
 _MAX_RETRIES = 16
 

app/templates.py

@@ -30,8 +30,8 @@ from app.config import VERSION
 from app.config import generate_csrf_token
 from app.config import session_serializer
 from app.database import AsyncSession
-from app.database import now
 from app.media import proxied_media_url
+from app.utils.datetime import now
 from app.utils.highlight import HIGHLIGHT_CSS
 from app.utils.highlight import highlight
 

app/templates/indieauth_flow.html

@@ -5,7 +5,7 @@
 <div style="display:flex">
 {% if client.logo %}
 <div style="flex:initial;width:100px;">
-<img src="{{client.logo}}" style="max-width:100px;">
+<img src="{{client.logo | media_proxy_url }}" style="max-width:100px;">
 </div>
 {% endif %}
 <div style="flex:1;">

app/templates/object.html

@@ -22,7 +22,7 @@
 {% macro display_replies_tree(replies_tree_node) %}
 
 {% if replies_tree_node.is_requested %}
-    {{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=replies_tree_node.ap_object.webmentions or [], expanded=not replies_tree_node.is_root) }}
+{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root) }}
 {% else %}
     {{ utils.display_object(replies_tree_node.ap_object) }}
 {% endif %}

app/templates/utils.html

@@ -388,9 +388,9 @@
             </li>
             {% endif %}
 
-            {% if object.webmentions %}
+            {% if object.webmentions_count %}
             <li>
-                <a href="{{ object.url }}"><strong>{{ object.webmentions | length }}</strong> webmention{{ object.webmentions | length | pluralize }}</a>
+                <a href="{{ object.url }}"><strong>{{ object.webmentions_count }}</strong> webmention{{ object.webmentions_count | pluralize }}</a>
             </li>
             {% endif %}
 
@@ -491,9 +491,12 @@
             <div style="flex: 0 1 30%;max-width: 50%;">Webmentions
                 <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
                 {% for webmention in webmentions %}
-                <a href="{{ webmention.url }}" title="{{ webmention.actor_name }}" style="height:50px;" rel="noreferrer">
+                    {% set wm = webmention.as_facepile_item %}
-                        <img src="{{ webmention.actor_icon_url | media_proxy_url }}" alt="{{ webmention.actor_name }}" style="max-width:50px;">
+                    {% if wm %}
-                    </a>
+                        <a href="{{ wm.url }}" title="{{ wm.actor_name }}" style="height:50px;" rel="noreferrer">
+                            <img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}" style="max-width:50px;">
+                        </a>
+                    {% endif %}
                 {% endfor %}
                 </div>
             </div>

app/utils/datetime.py

@@ -6,3 +6,7 @@ from dateutil.parser import isoparse
 
 def parse_isoformat(isodate: str) -> datetime:
     return isoparse(isodate).astimezone(timezone.utc)
+
+
+def now() -> datetime:
+    return datetime.now(timezone.utc)

app/utils/stats.py

@@ -13,7 +13,7 @@ from app import models
 from app.config import ROOT_DIR
 from app.database import AsyncSession
 from app.database import async_session
-from app.database import now
+from app.utils.datetime import now
 
 _DATA_DIR = ROOT_DIR / "data"
 

app/utils/webmentions.py

@@ -1,8 +1,13 @@
+from dataclasses import dataclass
+from typing import Any
+from typing import Optional
+
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from loguru import logger
 
 from app import config
+from app.utils.datetime import now
 from app.utils.url import is_url_valid
 from app.utils.url import make_abs
 
@@ -47,3 +52,38 @@ async def discover_webmention_endpoint(url: str) -> str | None:
     if not is_url_valid(wurl):
         return None
     return wurl
+
+
+@dataclass
+class Webmention:
+    actor_icon_url: str
+    actor_name: str
+    url: str
+    received_at: str
+
+    @classmethod
+    def from_microformats(
+        cls, items: list[dict[str, Any]], url: str
+    ) -> Optional["Webmention"]:
+        for item in items:
+            if item["type"][0] == "h-card":
+                return cls(
+                    actor_icon_url=make_abs(
+                        item["properties"]["photo"][0], url
+                    ),  # type: ignore
+                    actor_name=item["properties"]["name"][0],
+                    url=url,
+                    received_at=now().isoformat(),
+                )
+            if item["type"][0] == "h-entry":
+                author = item["properties"]["author"][0]
+                return cls(
+                    actor_icon_url=make_abs(
+                        author["properties"]["photo"][0], url
+                    ),  # type: ignore
+                    actor_name=author["properties"]["name"][0],
+                    url=url,
+                    received_at=now().isoformat(),
+                )
+
+        return None

app/webmentions.py

@@ -1,8 +1,3 @@
-from dataclasses import asdict
-from dataclasses import dataclass
-from typing import Any
-from typing import Optional
-
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
 from fastapi import Depends
@@ -10,54 +5,19 @@ from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
 from loguru import logger
+from sqlalchemy import select
 
+from app import models
 from app.boxes import get_outbox_object_by_ap_id
 from app.database import AsyncSession
 from app.database import get_db_session
-from app.database import now
 from app.utils import microformats
 from app.utils.url import check_url
 from app.utils.url import is_url_valid
-from app.utils.url import make_abs
 
 router = APIRouter()
 
 
-@dataclass
-class Webmention:
-    actor_icon_url: str
-    actor_name: str
-    url: str
-    received_at: str
-
-    @classmethod
-    def from_microformats(
-        cls, items: list[dict[str, Any]], url: str
-    ) -> Optional["Webmention"]:
-        for item in items:
-            if item["type"][0] == "h-card":
-                return cls(
-                    actor_icon_url=make_abs(
-                        item["properties"]["photo"][0], url
-                    ),  # type: ignore
-                    actor_name=item["properties"]["name"][0],
-                    url=url,
-                    received_at=now().isoformat(),
-                )
-            if item["type"][0] == "h-entry":
-                author = item["properties"]["author"][0]
-                return cls(
-                    actor_icon_url=make_abs(
-                        author["properties"]["photo"][0], url
-                    ),  # type: ignore
-                    actor_name=author["properties"]["name"][0],
-                    url=url,
-                    received_at=now().isoformat(),
-                )
-
-        return None
-
-
 def is_source_containing_target(source_html: str, target_url: str) -> bool:
     soup = BeautifulSoup(source_html, "html5lib")
     for link in soup.find_all("a"):
@@ -92,40 +52,64 @@ async def webmention_endpoint(
 
     logger.info(f"Received webmention {source=} {target=}")
 
+    existing_webmention_in_db = (
+        await db_session.execute(
+            select(models.Webmention).where(
+                models.Webmention.source == source,
+                models.Webmention.target == target,
+            )
+        )
+    ).scalar_one_or_none()
+    if existing_webmention_in_db:
+        logger.info("Found existing Webmention, will try to update or delete")
+
     mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
     if not mentioned_object:
         logger.info(f"Invalid target {target=}")
+
+        if existing_webmention_in_db:
+            logger.info("Deleting existing Webmention")
+            existing_webmention_in_db.is_deleted = True
+            await db_session.commit()
         raise HTTPException(status_code=400, detail="Invalid target")
 
     maybe_data_and_html = await microformats.fetch_and_parse(source)
     if not maybe_data_and_html:
         logger.info("failed to fetch source")
+
+        if existing_webmention_in_db:
+            logger.info("Deleting existing Webmention")
+            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
+            existing_webmention_in_db.is_deleted = True
+            await db_session.commit()
         raise HTTPException(status_code=400, detail="failed to fetch source")
 
     data, html = maybe_data_and_html
 
     if not is_source_containing_target(html, target):
         logger.warning("target not found in source")
+
+        if existing_webmention_in_db:
+            logger.info("Deleting existing Webmention")
+            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
+            existing_webmention_in_db.is_deleted = True
+            await db_session.commit()
+
         raise HTTPException(status_code=400, detail="target not found in source")
 
-    try:
+    if existing_webmention_in_db:
-        webmention = Webmention.from_microformats(data["items"], source)
+        existing_webmention_in_db.is_deleted = False
-        if not webmention:
+        existing_webmention_in_db.source_microformats = data
-            raise ValueError("Failed to fetch target data")
-    except Exception:
-        logger.warning("Failed build Webmention for {source=} with {data=}")
-        return JSONResponse(content={}, status_code=200)
-
-    logger.info(f"{webmention=}")
-
-    if mentioned_object.webmentions is None:
-        mentioned_object.webmentions = [asdict(webmention)]
     else:
-        mentioned_object.webmentions = [asdict(webmention)] + [
+        new_webmention = models.Webmention(
-            wm  # type: ignore
+            source=source,
-            for wm in mentioned_object.webmentions  # type: ignore
+            target=target,
-            if wm["url"] != source  # type: ignore
+            source_microformats=data,
-        ]
+            outbox_object_id=mentioned_object.id,
+        )
+        db_session.add(new_webmention)
+
+        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
 
     await db_session.commit()
 

scripts/build_docs.py

@@ -7,7 +7,7 @@ from jinja2 import select_autoescape
 from markdown import markdown
 
 from app.config import VERSION
-from app.database import now
+from app.utils.datetime import now
 
 
 def markdownify(content: str) -> str:

tests/factories.py

@@ -12,7 +12,7 @@ from app import models
 from app.actor import RemoteActor
 from app.ap_object import RemoteObject
 from app.database import SessionLocal
-from app.database import now
+from app.utils.datetime import now
 
 _Session = orm.scoped_session(SessionLocal)