Webmention improvements

- Tweak design for IndieAuth login flow
 - Webmentions notifications support
 - Refactor webmentions processing

alembic/env.py

@@ -49,6 +49,7 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        render_as_batch=True,
     )
 
     with context.begin_transaction():
@@ -69,7 +70,11 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

alembic/versions/2b51ae7047cb_webmention_notifications.py

@@ -0,0 +1,32 @@
+"""Webmention notifications
+
+Revision ID: 2b51ae7047cb
+Revises: e58c1ffadf2e
+Create Date: 2022-07-19 20:22:06.968951
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2b51ae7047cb'
+down_revision = 'e58c1ffadf2e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('notifications', schema=None) as batch_op:
+        batch_op.create_foreign_key('fk_webmention_id', 'webmention', ['webmention_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('notifications', schema=None) as batch_op:
+        batch_op.drop_constraint('fk_webmention_id', type_='foreignkey')
+
+    # ### end Alembic commands ###

app/admin.py

@@ -435,6 +435,7 @@ async def get_notifications(
                             models.OutboxObject.outbox_object_attachments
                         ).options(joinedload(models.OutboxObjectAttachment.upload)),
                     ),
+                    joinedload(models.Notification.webmention),
                 )
                 .order_by(models.Notification.created_at.desc())
             )

app/main.py

@@ -76,6 +76,8 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac
 # TODO(ts):
 #
 # Next:
+# - Webmention notification
+# - Page support
 # - Article support
 # - indieauth tweaks
 # - API for posting notes

app/models.py

@@ -300,35 +300,6 @@ class Following(Base):
     ap_actor_id = Column(String, nullable=False, unique=True)
 
 
-@enum.unique
-class NotificationType(str, enum.Enum):
-    NEW_FOLLOWER = "new_follower"
-    UNFOLLOW = "unfollow"
-    LIKE = "like"
-    UNDO_LIKE = "undo_like"
-    ANNOUNCE = "announce"
-    UNDO_ANNOUNCE = "undo_announce"
-    MENTION = "mention"
-
-
-class Notification(Base):
-    __tablename__ = "notifications"
-
-    id = Column(Integer, primary_key=True, index=True)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
-    notification_type = Column(Enum(NotificationType), nullable=True)
-    is_new = Column(Boolean, nullable=False, default=True)
-
-    actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
-    actor = relationship(Actor, uselist=False)
-
-    outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
-    outbox_object = relationship(OutboxObject, uselist=False)
-
-    inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
-    inbox_object = relationship(InboxObject, uselist=False)
-
-
 class IncomingActivity(Base):
     __tablename__ = "incoming_activity"
 
@@ -503,6 +474,43 @@ class Webmention(Base):
             return None
 
 
+@enum.unique
+class NotificationType(str, enum.Enum):
+    NEW_FOLLOWER = "new_follower"
+    UNFOLLOW = "unfollow"
+    LIKE = "like"
+    UNDO_LIKE = "undo_like"
+    ANNOUNCE = "announce"
+    UNDO_ANNOUNCE = "undo_announce"
+    MENTION = "mention"
+    NEW_WEBMENTION = "new_webmention"
+    UPDATED_WEBMENTION = "updated_webmention"
+    DELETED_WEBMENTION = "deleted_webmention"
+
+
+class Notification(Base):
+    __tablename__ = "notifications"
+
+    id = Column(Integer, primary_key=True, index=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+    notification_type = Column(Enum(NotificationType), nullable=True)
+    is_new = Column(Boolean, nullable=False, default=True)
+
+    actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
+    actor = relationship(Actor, uselist=False)
+
+    outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
+    outbox_object = relationship(OutboxObject, uselist=False)
+
+    inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
+    inbox_object = relationship(InboxObject, uselist=False)
+
+    webmention_id = Column(
+        Integer, ForeignKey("webmention.id", name="fk_webmention_id"), nullable=True
+    )
+    webmention = relationship(Webmention, uselist=False)
+
+
 outbox_fts = Table(
     "outbox_fts",
     metadata_obj,

app/templates/indieauth_flow.html

@@ -2,41 +2,40 @@
 {% extends "layout.html" %}
 {% block content %}
 <div class="box">
-<div style="display:flex">
+    <div style="display:flex;column-gap: 20px;">
-{% if client.logo %}
+        {% if client.logo %}
-<div style="flex:initial;width:100px;">
+        <div style="flex:initial;width:100px;">
-<img src="{{client.logo | media_proxy_url }}" style="max-width:100px;">
+            <img src="{{client.logo | media_proxy_url }}" style="max-width:100px;" alt="{{ client.name }} logo">
-</div>
+        </div>
-{% endif %}
+        {% endif %}
-<div style="flex:1;">
+        <div style="flex:1;">
-<div style="margin-top:20px">
+            <div style="padding-left: 20px;">
-<a class="lcolor" style="font-size:1.2em;font-weight:600;text-decoration:none;" href="{{ client.url }}">{{ client.name }}</a>
+                <a class="lcolor" style="font-size:1.2em;font-weight:600;" href="{{ client.url }}">{{ client.name }}</a>
-<p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
+                <p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
-</div>
-</div>
-</div>
 
-<form method="POST" action="{{ url_for('indieauth_flow') }}">
-    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-	{% if scopes %}
-	<h3>Scopes</h3>
-	<ul>
-	{% for scope in scopes %}
-	<li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
-	</li>
-	{% endfor %}
-	</ul>
-	{% endif %}
-	<input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
-	<input type="hidden" name="state" value="{{ state }}">
-	<input type="hidden" name="client_id" value="{{ client_id }}">
-	<input type="hidden" name="me" value="{{ me }}">
-	<input type="hidden" name="response_type" value="{{ response_type }}">
-	<input type="hidden" name="code_challenge" value="{{ code_challenge }}">
-	<input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
-	<input type="submit" value="login">
-</form>
 
-</div>
+                <form method="POST" action="{{ url_for('indieauth_flow') }}" class="form">
+                    {{ utils.embed_csrf_token() }}
+                    {% if scopes %}
+                    <h3>Scopes</h3>
+                    <ul>
+                    {% for scope in scopes %}
+                    <li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
+                    </li>
+                    {% endfor %}
+                    </ul>
+                    {% endif %}
+                    <input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
+                    <input type="hidden" name="state" value="{{ state }}">
+                    <input type="hidden" name="client_id" value="{{ client_id }}">
+                    <input type="hidden" name="me" value="{{ me }}">
+                    <input type="hidden" name="response_type" value="{{ response_type }}">
+                    <input type="hidden" name="code_challenge" value="{{ code_challenge }}">
+                    <input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
+                    <input type="submit" value="login">
+                </form>
+            </div>
+        </div>
+    </div>
 </div>
 {% endblock %}

app/templates/notifications.html

@@ -47,7 +47,36 @@
                     <a style="font-weight:bold;" href="{{ notif.actor.url }}">{{ notif.actor.display_name }}</a> mentioned you
                 </div>
                 {{ utils.display_object(notif.inbox_object) }}
- 
+            {% elif notif.notification_type.value == "new_webmention" %}
+                <div class="actor-action" title="{{ notif.created_at.isoformat() }}">
+                    new webmention from 
+                    {% set facepile_item = notif.webmention.as_facepile_item %}
+                    {% if facepile_item %}
+                        <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
+                    {% endif %}
+                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                </div>
+                {{ utils.display_object(notif.outbox_object) }}
+            {% elif notif.notification_type.value == "updated_webmention" %}
+                <div class="actor-action" title="{{ notif.created_at.isoformat() }}">
+                    updated webmention from 
+                    {% set facepile_item = notif.webmention.as_facepile_item %}
+                    {% if facepile_item %}
+                        <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
+                    {% endif %}
+                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                </div>
+                {{ utils.display_object(notif.outbox_object) }}
+            {% elif notif.notification_type.value == "deleted_webmention" %}
+                <div class="actor-action" title="{{ notif.created_at.isoformat() }}">
+                    deleted webmention from 
+                    {% set facepile_item = notif.webmention.as_facepile_item %}
+                    {% if facepile_item %}
+                        <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
+                    {% endif %}
+                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                </div>
+                {{ utils.display_object(notif.outbox_object) }}
             {% else %}
             <div class="actor-action">
                 Implement {{ notif.notification_type }}

app/utils/microformats.py

@@ -7,19 +7,28 @@ from loguru import logger
 from app import config
 
 
-async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str] | None:
+class URLNotFoundOrGone(Exception):
+    pass
+
+
+async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str]:
     async with httpx.AsyncClient() as client:
+        resp = await client.get(
+            url,
+            headers={
+                "User-Agent": config.USER_AGENT,
+            },
+            follow_redirects=True,
+        )
+        if resp.status_code in [404, 410]:
+            raise URLNotFoundOrGone
+
         try:
-            resp = await client.get(
-                url,
-                headers={
-                    "User-Agent": config.USER_AGENT,
-                },
-                follow_redirects=True,
-            )
             resp.raise_for_status()
-        except (httpx.HTTPError, httpx.HTTPStatusError):
+        except httpx.HTTPStatusError:
-            logger.exception(f"Failed to discover webmention endpoint for {url}")
+            logger.error(
-            return None
+                f"Failed to parse microformats for {url}: " f"got {resp.status_code}"
+            )
+            raise
 
     return mf2py.parse(doc=resp.text), resp.text

app/webmentions.py

@@ -1,3 +1,4 @@
+import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
 from fastapi import Depends
@@ -73,33 +74,53 @@ async def webmention_endpoint(
             await db_session.commit()
         raise HTTPException(status_code=400, detail="Invalid target")
 
-    maybe_data_and_html = await microformats.fetch_and_parse(source)
+    is_webmention_deleted = False
-    if not maybe_data_and_html:
+    try:
-        logger.info("failed to fetch source")
+        data_and_html = await microformats.fetch_and_parse(source)
+    except microformats.URLNotFoundOrGone:
+        is_webmention_deleted = True
+    except httpx.HTTPError:
+        raise HTTPException(status_code=500, detail=f"Fetch to process {source}")
 
+    data, html = data_and_html
+    is_target_found_in_source = is_source_containing_target(html, target)
+
+    data, html = data_and_html
+    if is_webmention_deleted or not is_target_found_in_source:
+        logger.warning(f"target {target=} not found in source")
         if existing_webmention_in_db:
             logger.info("Deleting existing Webmention")
             mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
             existing_webmention_in_db.is_deleted = True
-            await db_session.commit()
-        raise HTTPException(status_code=400, detail="failed to fetch source")
 
-    data, html = maybe_data_and_html
+            notif = models.Notification(
+                notification_type=models.NotificationType.DELETED_WEBMENTION,
+                outbox_object_id=mentioned_object.id,
+                webmention_id=existing_webmention_in_db.id,
+            )
+            db_session.add(notif)
 
-    if not is_source_containing_target(html, target):
-        logger.warning("target not found in source")
-
-        if existing_webmention_in_db:
-            logger.info("Deleting existing Webmention")
-            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
-            existing_webmention_in_db.is_deleted = True
             await db_session.commit()
 
-        raise HTTPException(status_code=400, detail="target not found in source")
+        if not is_target_found_in_source:
+            raise HTTPException(
+                status_code=400,
+                detail="target not found in source",
+            )
+        else:
+            return JSONResponse(content={}, status_code=200)
 
     if existing_webmention_in_db:
+        # Undelete if needed
         existing_webmention_in_db.is_deleted = False
         existing_webmention_in_db.source_microformats = data
+
+        notif = models.Notification(
+            notification_type=models.NotificationType.UPDATED_WEBMENTION,
+            outbox_object_id=mentioned_object.id,
+            webmention_id=existing_webmention_in_db.id,
+        )
+        db_session.add(notif)
     else:
         new_webmention = models.Webmention(
             source=source,
@@ -108,6 +129,14 @@ async def webmention_endpoint(
             outbox_object_id=mentioned_object.id,
         )
         db_session.add(new_webmention)
+        await db_session.flush()
+
+        notif = models.Notification(
+            notification_type=models.NotificationType.NEW_WEBMENTION,
+            outbox_object_id=mentioned_object.id,
+            webmention_id=new_webmention.id,
+        )
+        db_session.add(notif)
 
         mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1