Cleanup and improved webmentions support

This commit is contained in:
Thomas Sileo 2022-07-14 16:29:17 +02:00
parent 3abeab088f
commit c9aea8cab3
19 changed files with 231 additions and 83 deletions

View file

@ -0,0 +1,28 @@
"""Add webmentions count
Revision ID: 69ce9fbdc483
Revises: 1647cef23e9b
Create Date: 2022-07-14 15:35:01.716133
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '69ce9fbdc483'
down_revision = '1647cef23e9b'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('outbox', sa.Column('webmentions_count', sa.Integer(), server_default='0', nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('outbox', 'webmentions_count')
# ### end Alembic commands ###

View file

@ -0,0 +1,48 @@
"""Improved Webmentions
Revision ID: fd23d95e5c16
Revises: 69ce9fbdc483
Create Date: 2022-07-14 16:10:54.202455
"""
import sqlalchemy as sa
from sqlalchemy.dialects import sqlite
from alembic import op
# revision identifiers, used by Alembic.
revision = 'fd23d95e5c16'
down_revision = '69ce9fbdc483'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('webmention',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('is_deleted', sa.Boolean(), nullable=False),
sa.Column('source', sa.String(), nullable=False),
sa.Column('source_microformats', sa.JSON(), nullable=True),
sa.Column('target', sa.String(), nullable=False),
sa.Column('outbox_object_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['outbox_object_id'], ['outbox.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('source', 'target', name='uix_source_target')
)
op.create_index(op.f('ix_webmention_id'), 'webmention', ['id'], unique=False)
op.create_index(op.f('ix_webmention_source'), 'webmention', ['source'], unique=True)
op.create_index(op.f('ix_webmention_target'), 'webmention', ['target'], unique=False)
op.drop_column('outbox', 'webmentions')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('outbox', sa.Column('webmentions', sqlite.JSON(), nullable=True))
op.drop_index(op.f('ix_webmention_target'), table_name='webmention')
op.drop_index(op.f('ix_webmention_source'), table_name='webmention')
op.drop_index(op.f('ix_webmention_id'), table_name='webmention')
op.drop_table('webmention')
# ### end Alembic commands ###

View file

@ -27,12 +27,12 @@ from app.ap_object import RemoteObject
from app.config import BASE_URL from app.config import BASE_URL
from app.config import ID from app.config import ID
from app.database import AsyncSession from app.database import AsyncSession
from app.database import now
from app.outgoing_activities import new_outgoing_activity from app.outgoing_activities import new_outgoing_activity
from app.source import markdownify from app.source import markdownify
from app.uploads import upload_to_attachment from app.uploads import upload_to_attachment
from app.utils import opengraph from app.utils import opengraph
from app.utils import webmentions from app.utils import webmentions
from app.utils.datetime import now
from app.utils.datetime import parse_isoformat from app.utils.datetime import parse_isoformat
AnyboxObject = models.InboxObject | models.OutboxObject AnyboxObject = models.InboxObject | models.OutboxObject

View file

@ -1,4 +1,3 @@
import datetime
from typing import Any from typing import Any
from typing import AsyncGenerator from typing import AsyncGenerator
@ -23,10 +22,6 @@ async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit
Base: Any = declarative_base() Base: Any = declarative_base()
def now() -> datetime.datetime:
return datetime.datetime.now(datetime.timezone.utc)
async def get_db_session() -> AsyncGenerator[AsyncSession, None]: async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
async with async_session() as session: async with async_session() as session:
try: try:

View file

@ -13,7 +13,7 @@ from app import models
from app.boxes import save_to_inbox from app.boxes import save_to_inbox
from app.database import AsyncSession from app.database import AsyncSession
from app.database import async_session from app.database import async_session
from app.database import now from app.utils.datetime import now
_MAX_RETRIES = 5 _MAX_RETRIES = 5
@ -63,7 +63,7 @@ async def process_next_incoming_activity(db_session: AsyncSession) -> bool:
select(func.count(models.IncomingActivity.id)).where(*where) select(func.count(models.IncomingActivity.id)).where(*where)
) )
if q_count > 0: if q_count > 0:
logger.info(f"{q_count} outgoing activities ready to process") logger.info(f"{q_count} incoming activities ready to process")
if not q_count: if not q_count:
# logger.debug("No activities to process") # logger.debug("No activities to process")
return False return False

View file

@ -21,8 +21,8 @@ from app.admin import user_session_or_redirect
from app.config import verify_csrf_token from app.config import verify_csrf_token
from app.database import AsyncSession from app.database import AsyncSession
from app.database import get_db_session from app.database import get_db_session
from app.database import now
from app.utils import indieauth from app.utils import indieauth
from app.utils.datetime import now
router = APIRouter() router = APIRouter()

View file

@ -551,6 +551,17 @@ async def outbox_by_public_id(
.all() .all()
) )
webmentions = (
await db_session.scalars(
select(models.Webmention)
.filter(
models.Webmention.outbox_object_id == maybe_object.id,
models.Webmention.is_deleted.is_(False),
)
.limit(10)
)
).all()
return await templates.render_template( return await templates.render_template(
db_session, db_session,
request, request,
@ -560,6 +571,7 @@ async def outbox_by_public_id(
"outbox_object": maybe_object, "outbox_object": maybe_object,
"likes": likes, "likes": likes,
"shares": shares, "shares": shares,
"webmentions": webmentions,
}, },
) )

View file

@ -3,6 +3,7 @@ from typing import Any
from typing import Optional from typing import Optional
from typing import Union from typing import Union
from loguru import logger
from sqlalchemy import JSON from sqlalchemy import JSON
from sqlalchemy import Boolean from sqlalchemy import Boolean
from sqlalchemy import Column from sqlalchemy import Column
@ -22,7 +23,8 @@ from app.ap_object import Attachment
from app.ap_object import Object as BaseObject from app.ap_object import Object as BaseObject
from app.config import BASE_URL from app.config import BASE_URL
from app.database import Base from app.database import Base
from app.database import now from app.utils import webmentions
from app.utils.datetime import now
class Actor(Base, BaseActor): class Actor(Base, BaseActor):
@ -152,10 +154,11 @@ class OutboxObject(Base, BaseObject):
likes_count = Column(Integer, nullable=False, default=0) likes_count = Column(Integer, nullable=False, default=0)
announces_count = Column(Integer, nullable=False, default=0) announces_count = Column(Integer, nullable=False, default=0)
replies_count = Column(Integer, nullable=False, default=0) replies_count = Column(Integer, nullable=False, default=0)
webmentions_count: Mapped[int] = Column(
Integer, nullable=False, default=0, server_default="0"
)
# reactions: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True) # reactions: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
webmentions = Column(JSON, nullable=True)
og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True) og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
# For the featured collection # For the featured collection
@ -457,3 +460,34 @@ class IndieAuthAccessToken(Base):
expires_in = Column(Integer, nullable=False) expires_in = Column(Integer, nullable=False)
scope = Column(String, nullable=False) scope = Column(String, nullable=False)
is_revoked = Column(Boolean, nullable=False, default=False) is_revoked = Column(Boolean, nullable=False, default=False)
class Webmention(Base):
__tablename__ = "webmention"
__table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),)
id = Column(Integer, primary_key=True, index=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=now)
is_deleted = Column(Boolean, nullable=False, default=False)
source: Mapped[str] = Column(String, nullable=False, index=True, unique=True)
source_microformats: Mapped[dict[str, Any] | None] = Column(JSON, nullable=True)
target = Column(String, nullable=False, index=True)
outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
outbox_object = relationship(OutboxObject, uselist=False)
@property
def as_facepile_item(self) -> webmentions.Webmention | None:
if not self.source_microformats:
return None
try:
return webmentions.Webmention.from_microformats(
self.source_microformats["items"], self.source
)
except Exception:
logger.warning(
f"Failed to generate facefile item for Webmention id={self.id}"
)
return None

View file

@ -20,8 +20,8 @@ from app.actor import _actor_hash
from app.config import KEY_PATH from app.config import KEY_PATH
from app.database import AsyncSession from app.database import AsyncSession
from app.database import SessionLocal from app.database import SessionLocal
from app.database import now
from app.key import Key from app.key import Key
from app.utils.datetime import now
_MAX_RETRIES = 16 _MAX_RETRIES = 16

View file

@ -30,8 +30,8 @@ from app.config import VERSION
from app.config import generate_csrf_token from app.config import generate_csrf_token
from app.config import session_serializer from app.config import session_serializer
from app.database import AsyncSession from app.database import AsyncSession
from app.database import now
from app.media import proxied_media_url from app.media import proxied_media_url
from app.utils.datetime import now
from app.utils.highlight import HIGHLIGHT_CSS from app.utils.highlight import HIGHLIGHT_CSS
from app.utils.highlight import highlight from app.utils.highlight import highlight

View file

@ -5,7 +5,7 @@
<div style="display:flex"> <div style="display:flex">
{% if client.logo %} {% if client.logo %}
<div style="flex:initial;width:100px;"> <div style="flex:initial;width:100px;">
<img src="{{client.logo}}" style="max-width:100px;"> <img src="{{client.logo | media_proxy_url }}" style="max-width:100px;">
</div> </div>
{% endif %} {% endif %}
<div style="flex:1;"> <div style="flex:1;">

View file

@ -22,7 +22,7 @@
{% macro display_replies_tree(replies_tree_node) %} {% macro display_replies_tree(replies_tree_node) %}
{% if replies_tree_node.is_requested %} {% if replies_tree_node.is_requested %}
{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=replies_tree_node.ap_object.webmentions or [], expanded=not replies_tree_node.is_root) }} {{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root) }}
{% else %} {% else %}
{{ utils.display_object(replies_tree_node.ap_object) }} {{ utils.display_object(replies_tree_node.ap_object) }}
{% endif %} {% endif %}

View file

@ -388,9 +388,9 @@
</li> </li>
{% endif %} {% endif %}
{% if object.webmentions %} {% if object.webmentions_count %}
<li> <li>
<a href="{{ object.url }}"><strong>{{ object.webmentions | length }}</strong> webmention{{ object.webmentions | length | pluralize }}</a> <a href="{{ object.url }}"><strong>{{ object.webmentions_count }}</strong> webmention{{ object.webmentions_count | pluralize }}</a>
</li> </li>
{% endif %} {% endif %}
@ -491,9 +491,12 @@
<div style="flex: 0 1 30%;max-width: 50%;">Webmentions <div style="flex: 0 1 30%;max-width: 50%;">Webmentions
<div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;"> <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
{% for webmention in webmentions %} {% for webmention in webmentions %}
<a href="{{ webmention.url }}" title="{{ webmention.actor_name }}" style="height:50px;" rel="noreferrer"> {% set wm = webmention.as_facepile_item %}
<img src="{{ webmention.actor_icon_url | media_proxy_url }}" alt="{{ webmention.actor_name }}" style="max-width:50px;"> {% if wm %}
</a> <a href="{{ wm.url }}" title="{{ wm.actor_name }}" style="height:50px;" rel="noreferrer">
<img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}" style="max-width:50px;">
</a>
{% endif %}
{% endfor %} {% endfor %}
</div> </div>
</div> </div>

View file

@ -6,3 +6,7 @@ from dateutil.parser import isoparse
def parse_isoformat(isodate: str) -> datetime: def parse_isoformat(isodate: str) -> datetime:
return isoparse(isodate).astimezone(timezone.utc) return isoparse(isodate).astimezone(timezone.utc)
def now() -> datetime:
return datetime.now(timezone.utc)

View file

@ -13,7 +13,7 @@ from app import models
from app.config import ROOT_DIR from app.config import ROOT_DIR
from app.database import AsyncSession from app.database import AsyncSession
from app.database import async_session from app.database import async_session
from app.database import now from app.utils.datetime import now
_DATA_DIR = ROOT_DIR / "data" _DATA_DIR = ROOT_DIR / "data"

View file

@ -1,8 +1,13 @@
from dataclasses import dataclass
from typing import Any
from typing import Optional
import httpx import httpx
from bs4 import BeautifulSoup # type: ignore from bs4 import BeautifulSoup # type: ignore
from loguru import logger from loguru import logger
from app import config from app import config
from app.utils.datetime import now
from app.utils.url import is_url_valid from app.utils.url import is_url_valid
from app.utils.url import make_abs from app.utils.url import make_abs
@ -47,3 +52,38 @@ async def discover_webmention_endpoint(url: str) -> str | None:
if not is_url_valid(wurl): if not is_url_valid(wurl):
return None return None
return wurl return wurl
@dataclass
class Webmention:
actor_icon_url: str
actor_name: str
url: str
received_at: str
@classmethod
def from_microformats(
cls, items: list[dict[str, Any]], url: str
) -> Optional["Webmention"]:
for item in items:
if item["type"][0] == "h-card":
return cls(
actor_icon_url=make_abs(
item["properties"]["photo"][0], url
), # type: ignore
actor_name=item["properties"]["name"][0],
url=url,
received_at=now().isoformat(),
)
if item["type"][0] == "h-entry":
author = item["properties"]["author"][0]
return cls(
actor_icon_url=make_abs(
author["properties"]["photo"][0], url
), # type: ignore
actor_name=author["properties"]["name"][0],
url=url,
received_at=now().isoformat(),
)
return None

View file

@ -1,8 +1,3 @@
from dataclasses import asdict
from dataclasses import dataclass
from typing import Any
from typing import Optional
from bs4 import BeautifulSoup # type: ignore from bs4 import BeautifulSoup # type: ignore
from fastapi import APIRouter from fastapi import APIRouter
from fastapi import Depends from fastapi import Depends
@ -10,54 +5,19 @@ from fastapi import HTTPException
from fastapi import Request from fastapi import Request
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from loguru import logger from loguru import logger
from sqlalchemy import select
from app import models
from app.boxes import get_outbox_object_by_ap_id from app.boxes import get_outbox_object_by_ap_id
from app.database import AsyncSession from app.database import AsyncSession
from app.database import get_db_session from app.database import get_db_session
from app.database import now
from app.utils import microformats from app.utils import microformats
from app.utils.url import check_url from app.utils.url import check_url
from app.utils.url import is_url_valid from app.utils.url import is_url_valid
from app.utils.url import make_abs
router = APIRouter() router = APIRouter()
@dataclass
class Webmention:
actor_icon_url: str
actor_name: str
url: str
received_at: str
@classmethod
def from_microformats(
cls, items: list[dict[str, Any]], url: str
) -> Optional["Webmention"]:
for item in items:
if item["type"][0] == "h-card":
return cls(
actor_icon_url=make_abs(
item["properties"]["photo"][0], url
), # type: ignore
actor_name=item["properties"]["name"][0],
url=url,
received_at=now().isoformat(),
)
if item["type"][0] == "h-entry":
author = item["properties"]["author"][0]
return cls(
actor_icon_url=make_abs(
author["properties"]["photo"][0], url
), # type: ignore
actor_name=author["properties"]["name"][0],
url=url,
received_at=now().isoformat(),
)
return None
def is_source_containing_target(source_html: str, target_url: str) -> bool: def is_source_containing_target(source_html: str, target_url: str) -> bool:
soup = BeautifulSoup(source_html, "html5lib") soup = BeautifulSoup(source_html, "html5lib")
for link in soup.find_all("a"): for link in soup.find_all("a"):
@ -92,40 +52,64 @@ async def webmention_endpoint(
logger.info(f"Received webmention {source=} {target=}") logger.info(f"Received webmention {source=} {target=}")
existing_webmention_in_db = (
await db_session.execute(
select(models.Webmention).where(
models.Webmention.source == source,
models.Webmention.target == target,
)
)
).scalar_one_or_none()
if existing_webmention_in_db:
logger.info("Found existing Webmention, will try to update or delete")
mentioned_object = await get_outbox_object_by_ap_id(db_session, target) mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
if not mentioned_object: if not mentioned_object:
logger.info(f"Invalid target {target=}") logger.info(f"Invalid target {target=}")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="Invalid target") raise HTTPException(status_code=400, detail="Invalid target")
maybe_data_and_html = await microformats.fetch_and_parse(source) maybe_data_and_html = await microformats.fetch_and_parse(source)
if not maybe_data_and_html: if not maybe_data_and_html:
logger.info("failed to fetch source") logger.info("failed to fetch source")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="failed to fetch source") raise HTTPException(status_code=400, detail="failed to fetch source")
data, html = maybe_data_and_html data, html = maybe_data_and_html
if not is_source_containing_target(html, target): if not is_source_containing_target(html, target):
logger.warning("target not found in source") logger.warning("target not found in source")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="target not found in source") raise HTTPException(status_code=400, detail="target not found in source")
try: if existing_webmention_in_db:
webmention = Webmention.from_microformats(data["items"], source) existing_webmention_in_db.is_deleted = False
if not webmention: existing_webmention_in_db.source_microformats = data
raise ValueError("Failed to fetch target data")
except Exception:
logger.warning("Failed build Webmention for {source=} with {data=}")
return JSONResponse(content={}, status_code=200)
logger.info(f"{webmention=}")
if mentioned_object.webmentions is None:
mentioned_object.webmentions = [asdict(webmention)]
else: else:
mentioned_object.webmentions = [asdict(webmention)] + [ new_webmention = models.Webmention(
wm # type: ignore source=source,
for wm in mentioned_object.webmentions # type: ignore target=target,
if wm["url"] != source # type: ignore source_microformats=data,
] outbox_object_id=mentioned_object.id,
)
db_session.add(new_webmention)
mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
await db_session.commit() await db_session.commit()

View file

@ -7,7 +7,7 @@ from jinja2 import select_autoescape
from markdown import markdown from markdown import markdown
from app.config import VERSION from app.config import VERSION
from app.database import now from app.utils.datetime import now
def markdownify(content: str) -> str: def markdownify(content: str) -> str:

View file

@ -12,7 +12,7 @@ from app import models
from app.actor import RemoteActor from app.actor import RemoteActor
from app.ap_object import RemoteObject from app.ap_object import RemoteObject
from app.database import SessionLocal from app.database import SessionLocal
from app.database import now from app.utils.datetime import now
_Session = orm.scoped_session(SessionLocal) _Session = orm.scoped_session(SessionLocal)