Quick hotfix for retries

Also expand documentation about custom templates

AUTHORS

@@ -0,0 +1,11 @@
+Thomas Sileo <t@a4.io>
+Kevin Wallace <doof@doof.net>
+Miguel Jacq <mig@mig5.net>
+Alexey Shpakovsky <alexey@shpakovsky.ru>
+Josh Washburne <josh@jodh.us>
+João Costa <jdpc557@gmail.com>
+Sam <samr1.dev@pm.me>
+Ash McAllan <acegiak@gmail.com>
+Cassio Zen <cassio@hey.com>
+Cocoa <momijizukamori@gmail.com>
+Jane <jane@janeirl.dev>

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim as python-base
+FROM python:3.11-slim as python-base
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     POETRY_HOME="/opt/poetry" \
@@ -10,13 +10,18 @@ ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
 
 FROM python-base as builder-base
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends curl build-essential gcc 
+RUN apt-get install -y --no-install-recommends curl build-essential gcc libffi-dev libssl-dev libxml2-dev libxslt1-dev zlib1g-dev libxslt-dev gcc libjpeg-dev zlib1g-dev libwebp-dev
+# rustc is needed to compile Python packages
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
 RUN curl -sSL https://install.python-poetry.org | python3 - 
 WORKDIR $PYSETUP_PATH
 COPY poetry.lock pyproject.toml ./
-RUN poetry install --no-dev
+RUN poetry install --only main
 
 FROM python-base as production
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends libjpeg-dev libxslt1-dev libxml2-dev libxslt-dev
 RUN groupadd --gid 1000 microblogpub \
   && useradd --uid 1000 --gid microblogpub --shell /bin/bash microblogpub
 COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH

Makefile

@@ -9,12 +9,39 @@ build:
 config:
 	# Run and remove instantly
 	-docker run --rm -it --volume `pwd`/data:/app/data microblogpub/microblogpub inv configuration-wizard
-	-docker run --env MICROBLOGPUB_CONFIG_FILE=tests.toml --rm -it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv configuration-wizard
 
 .PHONY: update
 update:
-	-docker run --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv update
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv update --no-update-deps
 
 .PHONY: prune-old-data
 prune-old-data:
-	-docker run --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv prune-old-data
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv prune-old-data
+
+.PHONY: webfinger
+webfinger:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv webfinger $(account)
+
+.PHONY: move-to
+move-to:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv move-to $(account)
+
+.PHONY: self-destruct
+self-destruct:
+	-docker run --rm --it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv self-destruct
+
+.PHONY: reset-password
+reset-password:
+	-docker run --rm -it --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv reset-password
+
+.PHONY: check-config
+check-config:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv check-config
+
+.PHONY: compile-scss
+compile-scss:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv compile-scss
+
+.PHONY: import-mastodon-following-accounts 
+import-mastodon-following-accounts:
+	-docker run --rm --volume `pwd`/data:/app/data --volume `pwd`/app/static:/app/app/static microblogpub/microblogpub inv import-mastodon-following-accounts $(path)

README.md

@@ -10,6 +10,7 @@ Instances in the wild:
  - [microblog.pub](https://microblog.pub/) (follow to get updated about the project)
  - [hexa.ninja](https://hexa.ninja) (theme customization example)
  - [testing.microblog.pub](https://testing.microblog.pub/)
+ - [Irish Left Archive](https://posts.leftarchive.ie/) (another theme customization example)
 
 There are still some rough edges, but the server is mostly functional.
 
@@ -22,7 +23,7 @@ There are still some rough edges, but the server is mostly functional.
     - Author notes in Markdown, with code highlighting support
     - Dedicated section for articles/blog posts (enabled when the first article is posted)
  - Lightweight
-    - Uses SQLite, and no external dependencies except Python 3.10+
+    - Uses SQLite, and Python 3.10+
     - Can be deployed on small VPS
  - Privacy-aware
     - EXIF metadata (like GPS location) are stripped before storage
@@ -58,7 +59,7 @@ All the development takes place on [sourcehut](https://sr.ht/~tsileo/microblog.p
  - [Issue tracker](https://todo.sr.ht/~tsileo/microblog.pub)
  - [Mailing list](https://sr.ht/~tsileo/microblog.pub/lists)
 
-Contributions are welcomed, check out the [documentation](https://docs.microblog.pub) for more details.
+Contributions are welcomed, check out the [contributing section of the documentation](https://docs.microblog.pub/developer_guide.html#contributing) for more details.
 
 
 ## License

alembic/versions/2022_10_30_1409-b28c0551c236_add_a_slug_field_for_outbox_objects.py

@@ -0,0 +1,48 @@
+"""Add a slug field for outbox objects
+
+Revision ID: b28c0551c236
+Revises: 604d125ea2fb
+Create Date: 2022-10-30 14:09:14.540461+00:00
+
+"""
+import sqlalchemy as sa
+from sqlalchemy import select
+from sqlalchemy.orm.session import Session
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'b28c0551c236'
+down_revision = '604d125ea2fb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('outbox', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('slug', sa.String(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_outbox_slug'), ['slug'], unique=False)
+
+    # ### end Alembic commands ###
+
+    # Backfill the slug for existing articles
+    from app.models import OutboxObject
+    from app.utils.text import slugify
+    sess = Session(op.get_bind())
+    articles = sess.execute(select(OutboxObject).where(
+        OutboxObject.ap_type == "Article")
+    ).scalars()
+    for article in articles:
+        title = article.ap_object["name"]
+        article.slug = slugify(title)
+    sess.commit()
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('outbox', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_outbox_slug'))
+        batch_op.drop_column('slug')
+
+    # ### end Alembic commands ###

alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py

@@ -0,0 +1,32 @@
+"""Add Webmention.webmention_type
+
+Revision ID: fadfd359ce78
+Revises: b28c0551c236
+Create Date: 2022-11-16 19:42:56.925512+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'fadfd359ce78'
+down_revision = 'b28c0551c236'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('webmention_type', sa.Enum('UNKNOWN', 'LIKE', 'REPLY', 'REPOST', name='webmentiontype'), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('webmention', schema=None) as batch_op:
+        batch_op.drop_column('webmention_type')
+
+    # ### end Alembic commands ###

alembic/versions/2022_12_12_1926-9b404c47970a_add_option_to_hide_announces_from_actor.py

@@ -0,0 +1,32 @@
+"""Add option to hide announces from actor
+
+Revision ID: 9b404c47970a
+Revises: fadfd359ce78
+Create Date: 2022-12-12 19:26:36.912763+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '9b404c47970a'
+down_revision = 'fadfd359ce78'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('actor', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('are_announces_hidden_from_stream', sa.Boolean(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('actor', schema=None) as batch_op:
+        batch_op.drop_column('are_announces_hidden_from_stream')
+
+    # ### end Alembic commands ###

alembic/versions/2022_12_16_1730-4ab54becec04_add_oauth_client.py

@@ -0,0 +1,48 @@
+"""Add OAuth client
+
+Revision ID: 4ab54becec04
+Revises: 9b404c47970a
+Create Date: 2022-12-16 17:30:54.520477+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '4ab54becec04'
+down_revision = '9b404c47970a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('oauth_client',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('client_name', sa.String(), nullable=False),
+    sa.Column('redirect_uris', sa.JSON(), nullable=True),
+    sa.Column('client_uri', sa.String(), nullable=True),
+    sa.Column('logo_uri', sa.String(), nullable=True),
+    sa.Column('scope', sa.String(), nullable=True),
+    sa.Column('client_id', sa.String(), nullable=False),
+    sa.Column('client_secret', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('client_secret')
+    )
+    with op.batch_alter_table('oauth_client', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_oauth_client_client_id'), ['client_id'], unique=True)
+        batch_op.create_index(batch_op.f('ix_oauth_client_id'), ['id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('oauth_client', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_oauth_client_id'))
+        batch_op.drop_index(batch_op.f('ix_oauth_client_client_id'))
+
+    op.drop_table('oauth_client')
+    # ### end Alembic commands ###

alembic/versions/2022_12_18_1126-a209f0333f5a_add_oauth_refresh_token_support.py

@@ -0,0 +1,36 @@
+"""Add OAuth refresh token support
+
+Revision ID: a209f0333f5a
+Revises: 4ab54becec04
+Create Date: 2022-12-18 11:26:31.976348+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'a209f0333f5a'
+down_revision = '4ab54becec04'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('indieauth_access_token', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('refresh_token', sa.String(), nullable=True))
+        batch_op.add_column(sa.Column('was_refreshed', sa.Boolean(), server_default='0', nullable=False))
+        batch_op.create_index(batch_op.f('ix_indieauth_access_token_refresh_token'), ['refresh_token'], unique=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('indieauth_access_token', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_indieauth_access_token_refresh_token'))
+        batch_op.drop_column('was_refreshed')
+        batch_op.drop_column('refresh_token')
+
+    # ### end Alembic commands ###

app/activitypub.py

@@ -6,12 +6,15 @@ from typing import Any
 
 import httpx
 from loguru import logger
-from markdown import markdown
 
 from app import config
+from app.config import ALSO_KNOWN_AS
 from app.config import AP_CONTENT_TYPE  # noqa: F401
+from app.config import MOVED_TO
 from app.httpsig import auth
 from app.key import get_pubkey_as_pem
+from app.source import dedup_tags
+from app.source import hashtagify
 from app.utils.url import check_url
 
 if TYPE_CHECKING:
@@ -32,6 +35,7 @@ AS_EXTENDED_CTX = [
         "sensitive": "as:sensitive",
         "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
         "alsoKnownAs": {"@id": "as:alsoKnownAs", "@type": "@id"},
+        "movedTo": {"@id": "as:movedTo", "@type": "@id"},
         # toot
         "toot": "http://joinmastodon.org/ns#",
         "featured": {"@id": "toot:featured", "@type": "@id"},
@@ -49,11 +53,26 @@ AS_EXTENDED_CTX = [
 ]
 
 
-class ObjectIsGoneError(Exception):
+class FetchError(Exception):
+    def __init__(self, url: str, resp: httpx.Response | None = None) -> None:
+        resp_part = ""
+        if resp:
+            resp_part = f", got HTTP {resp.status_code}: {resp.text}"
+        message = f"Failed to fetch {url}{resp_part}"
+        super().__init__(message)
+        self.resp = resp
+        self.url = url
+
+
+class ObjectIsGoneError(FetchError):
     pass
 
 
-class ObjectNotFoundError(Exception):
+class ObjectNotFoundError(FetchError):
+    pass
+
+
+class ObjectUnavailableError(FetchError):
     pass
 
 
@@ -81,6 +100,21 @@ class VisibilityEnum(str, enum.Enum):
         }[key]
 
 
+_LOCAL_ACTOR_SUMMARY, _LOCAL_ACTOR_TAGS = hashtagify(config.CONFIG.summary)
+_LOCAL_ACTOR_METADATA = []
+if config.CONFIG.metadata:
+    for kv in config.CONFIG.metadata:
+        kv_value, kv_tags = hashtagify(kv.value)
+        _LOCAL_ACTOR_METADATA.append(
+            {
+                "name": kv.key,
+                "type": "PropertyValue",
+                "value": kv_value,
+            }
+        )
+        _LOCAL_ACTOR_TAGS.extend(kv_tags)
+
+
 ME = {
     "@context": AS_EXTENDED_CTX,
     "type": "Person",
@@ -92,7 +126,7 @@ ME = {
     "outbox": config.BASE_URL + "/outbox",
     "preferredUsername": config.USERNAME,
     "name": config.CONFIG.name,
-    "summary": config.CONFIG.summary,
+    "summary": _LOCAL_ACTOR_SUMMARY,
     "endpoints": {
         # For compat with servers expecting a sharedInbox...
         "sharedInbox": config.BASE_URL
@@ -100,28 +134,35 @@ ME = {
     },
     "url": config.ID + "/",  # XXX: the path is important for Mastodon compat
     "manuallyApprovesFollowers": config.CONFIG.manually_approves_followers,
-    "attachment": [
-        {
-            "name": kv.key,
-            "type": "PropertyValue",
-            "value": markdown(kv.value, extensions=["mdx_linkify", "fenced_code"]),
-        }
-        for kv in config.CONFIG.metadata
-    ]
-    if config.CONFIG.metadata
-    else [],
-    "icon": {
-        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
-        "type": "Image",
-        "url": config.CONFIG.icon_url,
-    },
+    "attachment": _LOCAL_ACTOR_METADATA,
     "publicKey": {
         "id": f"{config.ID}#main-key",
         "owner": config.ID,
         "publicKeyPem": get_pubkey_as_pem(config.KEY_PATH),
     },
+    "tag": dedup_tags(_LOCAL_ACTOR_TAGS),
 }
 
+if config.CONFIG.icon_url:
+    ME["icon"] = {
+        "mediaType": mimetypes.guess_type(config.CONFIG.icon_url)[0],
+        "type": "Image",
+        "url": config.CONFIG.icon_url,
+    }
+
+if ALSO_KNOWN_AS:
+    ME["alsoKnownAs"] = [ALSO_KNOWN_AS]
+
+if MOVED_TO:
+    ME["movedTo"] = MOVED_TO
+
+if config.CONFIG.image_url:
+    ME["image"] = {
+        "mediaType": mimetypes.guess_type(config.CONFIG.image_url)[0],
+        "type": "Image",
+        "url": config.CONFIG.image_url,
+    }
+
 
 class NotAnObjectError(Exception):
     def __init__(self, url: str, resp: httpx.Response | None = None) -> None:
@@ -153,11 +194,17 @@ async def fetch(
 
     # Special handling for deleted object
     if resp.status_code == 410:
-        raise ObjectIsGoneError(f"{url} is gone")
+        raise ObjectIsGoneError(url, resp)
+    elif resp.status_code in [401, 403]:
+        raise ObjectUnavailableError(url, resp)
     elif resp.status_code == 404:
-        raise ObjectNotFoundError(f"{url} not found")
+        raise ObjectNotFoundError(url, resp)
 
+    try:
         resp.raise_for_status()
+    except httpx.HTTPError as http_error:
+        raise FetchError(url, resp) from http_error
+
     try:
         return resp.json()
     except json.JSONDecodeError:

app/actor.py

@@ -1,16 +1,25 @@
 import hashlib
 import typing
 from dataclasses import dataclass
+from datetime import timedelta
 from functools import cached_property
 from typing import Union
 from urllib.parse import urlparse
 
+import httpx
+from loguru import logger
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
 from app import activitypub as ap
 from app import media
+from app.config import BASE_URL
+from app.config import USER_AGENT
+from app.config import USERNAME
+from app.config import WEBFINGER_DOMAIN
 from app.database import AsyncSession
+from app.utils.datetime import as_utc
+from app.utils.datetime import now
 
 if typing.TYPE_CHECKING:
     from app.models import Actor as ActorModel
@@ -22,7 +31,38 @@ def _handle(raw_actor: ap.RawObject) -> str:
     if not domain.hostname:
         raise ValueError(f"Invalid actor ID {ap_id}")
 
-    return f'@{raw_actor["preferredUsername"]}@{domain.hostname}'  # type: ignore
+    handle = f'@{raw_actor["preferredUsername"]}@{domain.hostname}'  # type: ignore
+
+    # TODO: cleanup this
+    # Next, check for custom webfinger domains
+    resp: httpx.Response | None = None
+    for url in {
+        f"https://{domain.hostname}/.well-known/webfinger",
+        f"http://{domain.hostname}/.well-known/webfinger",
+    }:
+        try:
+            logger.info(f"Webfinger {handle} at {url}")
+            resp = httpx.get(
+                url,
+                params={"resource": f"acct:{handle[1:]}"},
+                headers={
+                    "User-Agent": USER_AGENT,
+                },
+                follow_redirects=True,
+            )
+            resp.raise_for_status()
+            break
+        except Exception:
+            logger.exception(f"Failed to webfinger {handle}")
+
+    if resp:
+        try:
+            json_resp = resp.json()
+            if json_resp.get("subject", "").startswith("acct:"):
+                return "@" + json_resp["subject"].removeprefix("acct:")
+        except Exception:
+            logger.exception(f"Failed to parse webfinger response for {handle}")
+    return handle
 
 
 class Actor:
@@ -56,7 +96,7 @@ class Actor:
             return self.name
         return self.preferred_username
 
-    @property
+    @cached_property
     def handle(self) -> str:
         return _handle(self.ap_actor)
 
@@ -78,11 +118,21 @@ class Actor:
 
     @property
     def icon_url(self) -> str | None:
-        return self.ap_actor.get("icon", {}).get("url")
+        if icon := self.ap_actor.get("icon"):
+            return icon.get("url")
+        return None
 
     @property
     def icon_media_type(self) -> str | None:
-        return self.ap_actor.get("icon", {}).get("mediaType")
+        if icon := self.ap_actor.get("icon"):
+            return icon.get("mediaType")
+        return None
+
+    @property
+    def image_url(self) -> str | None:
+        if image := self.ap_actor.get("image"):
+            return image.get("url")
+        return None
 
     @property
     def public_key_as_pem(self) -> str:
@@ -97,18 +147,18 @@ class Actor:
         if self.icon_url:
             return media.proxied_media_url(self.icon_url)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def resized_icon_url(self) -> str:
         if self.icon_url:
             return media.resized_media_url(self.icon_url, 50)
         else:
-            return "/static/nopic.png"
+            return BASE_URL + "/static/nopic.png"
 
     @property
     def tags(self) -> list[ap.RawObject]:
-        return self.ap_actor.get("tag", [])
+        return ap.as_list(self.ap_actor.get("tag", []))
 
     @property
     def followers_collection_id(self) -> str | None:
@@ -118,19 +168,28 @@ class Actor:
     def attachments(self) -> list[ap.RawObject]:
         return ap.as_list(self.ap_actor.get("attachment", []))
 
+    @cached_property
+    def moved_to(self) -> str | None:
+        return self.ap_actor.get("movedTo")
+
     @cached_property
     def server(self) -> str:
         return urlparse(self.ap_id).hostname  # type: ignore
 
 
 class RemoteActor(Actor):
-    def __init__(self, ap_actor: ap.RawObject) -> None:
+    def __init__(self, ap_actor: ap.RawObject, handle: str | None = None) -> None:
         if (ap_type := ap_actor.get("type")) not in ap.ACTOR_TYPES:
             raise ValueError(f"Unexpected actor type: {ap_type}")
 
         self._ap_actor = ap_actor
         self._ap_type = ap_type
 
+        if handle is None:
+            handle = _handle(ap_actor)
+
+        self._handle = handle
+
     @property
     def ap_actor(self) -> ap.RawObject:
         return self._ap_actor
@@ -143,8 +202,12 @@ class RemoteActor(Actor):
     def is_from_db(self) -> bool:
         return False
 
+    @property
+    def handle(self) -> str:
+        return self._handle
 
-LOCAL_ACTOR = RemoteActor(ap_actor=ap.ME)
+
+LOCAL_ACTOR = RemoteActor(ap_actor=ap.ME, handle=f"@{USERNAME}@{WEBFINGER_DOMAIN}")
 
 
 async def save_actor(db_session: AsyncSession, ap_actor: ap.RawObject) -> "ActorModel":
@@ -154,9 +217,9 @@ async def save_actor(db_session: AsyncSession, ap_actor: ap.RawObject) -> "Actor
         raise ValueError(f"Invalid type {ap_type} for actor {ap_actor}")
 
     actor = models.Actor(
-        ap_id=ap_actor["id"],
+        ap_id=ap.get_id(ap_actor["id"]),
         ap_actor=ap_actor,
-        ap_type=ap_actor["type"],
+        ap_type=ap.as_list(ap_actor["type"])[0],
         handle=_handle(ap_actor),
     )
     db_session.add(actor)
@@ -184,13 +247,64 @@ async def fetch_actor(
     if existing_actor:
         if existing_actor.is_deleted:
             raise ap.ObjectNotFoundError(f"{actor_id} was deleted")
+
+        if now() - as_utc(existing_actor.updated_at) > timedelta(hours=24):
+            logger.info(
+                f"Refreshing {actor_id=} last updated {existing_actor.updated_at}"
+            )
+            try:
+                ap_actor = await ap.fetch(actor_id)
+                await update_actor_if_needed(
+                    db_session,
+                    existing_actor,
+                    RemoteActor(ap_actor),
+                )
+                return existing_actor
+            except Exception:
+                logger.exception(f"Failed to refresh {actor_id}")
+                # If we fail to refresh the actor, return the cached one
                 return existing_actor
         else:
+            return existing_actor
+
     if save_if_not_found:
         ap_actor = await ap.fetch(actor_id)
+        # Some softwares uses URL when we expect ID or uses a different casing
+        # (like Birdsite LIVE) , which mean we may already have it in DB
+        existing_actor_by_url = (
+            await db_session.scalars(
+                select(models.Actor).where(
+                    models.Actor.ap_id == ap.get_id(ap_actor),
+                )
+            )
+        ).one_or_none()
+        if existing_actor_by_url:
+            # Update the actor as we had to fetch it anyway
+            await update_actor_if_needed(
+                db_session,
+                existing_actor_by_url,
+                RemoteActor(ap_actor),
+            )
+            return existing_actor_by_url
+
         return await save_actor(db_session, ap_actor)
     else:
-            raise ap.ObjectNotFoundError
+        raise ap.ObjectNotFoundError(actor_id)
+
+
+async def update_actor_if_needed(
+    db_session: AsyncSession,
+    actor_in_db: "ActorModel",
+    ra: RemoteActor,
+) -> None:
+    # Check if we actually need to udpte the actor in DB
+    if _actor_hash(ra) != _actor_hash(actor_in_db):
+        actor_in_db.ap_actor = ra.ap_actor
+        actor_in_db.handle = ra.handle
+        actor_in_db.ap_type = ra.ap_type
+
+    actor_in_db.updated_at = now()
+    await db_session.flush()
 
 
 @dataclass
@@ -199,8 +313,11 @@ class ActorMetadata:
     is_following: bool
     is_follower: bool
     is_follow_request_sent: bool
+    is_follow_request_rejected: bool
     outbox_follow_ap_id: str | None
     inbox_follow_ap_id: str | None
+    moved_to: typing.Optional["ActorModel"]
+    has_blocked_local_actor: bool
 
 
 ActorsMetadata = dict[str, ActorMetadata]
@@ -243,17 +360,57 @@ async def get_actors_metadata(
             )
         )
     }
+    rejected_follow_requests = {
+        reject.activity_object_ap_id
+        for reject in await db_session.execute(
+            select(models.InboxObject.activity_object_ap_id).where(
+                models.InboxObject.ap_type == "Reject",
+                models.InboxObject.ap_actor_id.in_(ap_actor_ids),
+            )
+        )
+    }
+    blocks = {
+        block.ap_actor_id
+        for block in await db_session.execute(
+            select(models.InboxObject.ap_actor_id).where(
+                models.InboxObject.ap_type == "Block",
+                models.InboxObject.undone_by_inbox_object_id.is_(None),
+                models.InboxObject.ap_actor_id.in_(ap_actor_ids),
+            )
+        )
+    }
+
     idx: ActorsMetadata = {}
     for actor in actors:
         if not actor.ap_id:
             raise ValueError("Should never happen")
+        moved_to = None
+        if actor.moved_to:
+            try:
+                moved_to = await fetch_actor(
+                    db_session,
+                    actor.moved_to,
+                    save_if_not_found=False,
+                )
+            except ap.ObjectNotFoundError:
+                pass
+            except Exception:
+                logger.exception(f"Failed to fetch {actor.moved_to=}")
+
         idx[actor.ap_id] = ActorMetadata(
             ap_actor_id=actor.ap_id,
             is_following=actor.ap_id in following,
             is_follower=actor.ap_id in followers,
             is_follow_request_sent=actor.ap_id in sent_follow_requests,
+            is_follow_request_rejected=bool(
+                sent_follow_requests[actor.ap_id] in rejected_follow_requests
+            )
+            if actor.ap_id in sent_follow_requests
+            else False,
             outbox_follow_ap_id=sent_follow_requests.get(actor.ap_id),
             inbox_follow_ap_id=followers.get(actor.ap_id),
+            moved_to=moved_to,
+            has_blocked_local_actor=actor.ap_id in blocks,
         )
     return idx
 
@@ -278,6 +435,9 @@ def _actor_hash(actor: Actor) -> bytes:
     if actor.icon_url:
         h.update(actor.icon_url.encode())
 
+    if actor.image_url:
+        h.update(actor.image_url.encode())
+
     if actor.attachments:
         for a in actor.attachments:
             if a.get("type") != "PropertyValue":
@@ -289,4 +449,7 @@ def _actor_hash(actor: Actor) -> bytes:
     h.update(actor.public_key_id.encode())
     h.update(actor.public_key_as_pem.encode())
 
+    if actor.moved_to:
+        h.update(actor.moved_to.encode())
+
     return h.digest()

app/admin.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from urllib.parse import quote
 
 import httpx
 from fastapi import APIRouter
@@ -11,6 +12,7 @@ from fastapi.exceptions import HTTPException
 from fastapi.responses import RedirectResponse
 from loguru import logger
 from sqlalchemy import and_
+from sqlalchemy import delete
 from sqlalchemy import func
 from sqlalchemy import or_
 from sqlalchemy import select
@@ -25,8 +27,11 @@ from app.actor import fetch_actor
 from app.actor import get_actors_metadata
 from app.boxes import get_inbox_object_by_ap_id
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import send_block
 from app.boxes import send_follow
+from app.boxes import send_unblock
 from app.config import EMOJIS
+from app.config import SESSION_TIMEOUT
 from app.config import generate_csrf_token
 from app.config import session_serializer
 from app.config import verify_csrf_token
@@ -34,29 +39,44 @@ from app.config import verify_password
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.lookup import lookup
+from app.templates import is_current_user_admin
 from app.uploads import save_upload
 from app.utils import pagination
 from app.utils.emoji import EMOJIS_BY_NAME
 
 
-def user_session_or_redirect(
+async def user_session_or_redirect(
     request: Request,
     session: str | None = Cookie(default=None),
 ) -> None:
+    if request.method == "POST":
+        form_data = await request.form()
+        if "redirect_url" in form_data:
+            redirect_url = form_data["redirect_url"]
+        else:
+            redirect_url = request.url_for("admin_stream")
+    else:
+        redirect_url = str(request.url)
+
     _RedirectToLoginPage = HTTPException(
         status_code=302,
-        headers={"Location": request.url_for("login") + f"?redirect={request.url}"},
+        headers={
+            "Location": request.url_for("login") + f"?redirect={quote(redirect_url)}"
+        },
     )
 
     if not session:
+        logger.info("No existing admin session")
         raise _RedirectToLoginPage
 
     try:
-        loaded_session = session_serializer.loads(session, max_age=3600 * 12)
+        loaded_session = session_serializer.loads(session, max_age=SESSION_TIMEOUT)
     except Exception:
+        logger.exception("Failed to validate admin session")
         raise _RedirectToLoginPage
 
     if not loaded_session.get("is_logged_in"):
+        logger.info(f"Admin session invalidated: {loaded_session}")
         raise _RedirectToLoginPage
 
     return None
@@ -68,16 +88,6 @@ router = APIRouter(
 unauthenticated_router = APIRouter()
 
 
-@router.get("/")
-async def admin_index(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-) -> templates.TemplateResponse:
-    return await templates.render_template(
-        db_session, request, "index.html", {"request": request}
-    )
-
-
 @router.get("/lookup")
 async def get_lookup(
     request: Request,
@@ -94,6 +104,8 @@ async def get_lookup(
             error = ap.FetchErrorTypeEnum.TIMEOUT
         except (ap.ObjectNotFoundError, ap.ObjectIsGoneError):
             error = ap.FetchErrorTypeEnum.NOT_FOUND
+        except (ap.ObjectUnavailableError):
+            error = ap.FetchErrorTypeEnum.UNAUHTORIZED
         except Exception:
             logger.exception(f"Failed to lookup {query}")
             error = ap.FetchErrorTypeEnum.INTERNAL_ERROR
@@ -122,7 +134,9 @@ async def get_lookup(
                 )
                 if requested_object:
                     return RedirectResponse(
-                        request.url_for("admin_object") + f"?ap_id={ap_object.ap_id}",
+                        request.url_for("admin_object")
+                        + f"?ap_id={ap_object.ap_id}#"
+                        + requested_object.permalink_id,
                         status_code=302,
                     )
 
@@ -175,8 +189,11 @@ async def admin_new(
             content += f"{in_reply_to_object.actor.handle} "
         for tag in in_reply_to_object.tags:
             if tag.get("type") == "Mention" and tag["name"] != LOCAL_ACTOR.handle:
+                try:
                     mentioned_actor = await fetch_actor(db_session, tag["href"])
                     content += f"{mentioned_actor.handle} "
+                except Exception:
+                    logger.exception(f"Failed to lookup {mentioned_actor}")
 
         # Copy the content warning if any
         if in_reply_to_object.summary:
@@ -211,6 +228,7 @@ async def admin_bookmarks(
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
+    # TODO: support pagination
     stream = (
         (
             await db_session.scalars(
@@ -335,6 +353,7 @@ async def admin_inbox(
                 "Update",
                 "Undo",
                 "Read",
+                "Reject",
                 "Add",
                 "Remove",
                 "EmojiReact",
@@ -431,6 +450,7 @@ async def admin_direct_messages(
                     models.InboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.InboxObject.is_transient.is_(False),
+                    models.InboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.InboxObject.ap_context, models.InboxObject.actor_id)
             )
@@ -453,6 +473,7 @@ async def admin_direct_messages(
                     models.OutboxObject.ap_context.is_not(None),
                     # Skip transient object like poll relies
                     models.OutboxObject.is_transient.is_(False),
+                    models.OutboxObject.is_deleted.is_(False),
                 )
                 .group_by(models.OutboxObject.ap_context)
             )
@@ -667,15 +688,30 @@ async def admin_outbox(
 
 @router.get("/notifications")
 async def get_notifications(
-    request: Request, db_session: AsyncSession = Depends(get_db_session)
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    cursor: str | None = None,
 ) -> templates.TemplateResponse:
+    where = []
+    if cursor:
+        decoded_cursor = pagination.decode_cursor(cursor)
+        where.append(models.Notification.created_at < decoded_cursor)
+
+    page_size = 20
+    remaining_count = await db_session.scalar(
+        select(func.count(models.Notification.id)).where(*where)
+    )
+
     notifications = (
         (
             await db_session.scalars(
                 select(models.Notification)
+                .where(*where)
                 .options(
                     joinedload(models.Notification.actor),
-                    joinedload(models.Notification.inbox_object),
+                    joinedload(models.Notification.inbox_object).options(
+                        joinedload(models.InboxObject.actor)
+                    ),
                     joinedload(models.Notification.outbox_object).options(
                         joinedload(
                             models.OutboxObject.outbox_object_attachments
@@ -684,6 +720,7 @@ async def get_notifications(
                     joinedload(models.Notification.webmention),
                 )
                 .order_by(models.Notification.created_at.desc())
+                .limit(page_size)
             )
         )
         .unique()
@@ -692,21 +729,42 @@ async def get_notifications(
     actors_metadata = await get_actors_metadata(
         db_session, [notif.actor for notif in notifications if notif.actor]
     )
+    more_unread_count = 0
+    next_cursor = None
 
-    for notif in notifications:
-        notif.is_new = False
-    await db_session.commit()
+    if notifications and remaining_count > page_size:
+        decoded_next_cursor = notifications[-1].created_at
+        next_cursor = pagination.encode_cursor(decoded_next_cursor)
 
-    return await templates.render_template(
+        # If on the "see more" page there's more unread notification, we want
+        # to display it next to the link
+        more_unread_count = await db_session.scalar(
+            select(func.count(models.Notification.id)).where(
+                models.Notification.is_new.is_(True),
+                models.Notification.created_at < decoded_next_cursor,
+            )
+        )
+
+    # Render the template before we change the new flag on notifications
+    tpl_resp = await templates.render_template(
         db_session,
         request,
         "notifications.html",
         {
             "notifications": notifications,
             "actors_metadata": actors_metadata,
+            "next_cursor": next_cursor,
+            "more_unread_count": more_unread_count,
         },
     )
 
+    if len({notif.id for notif in notifications if notif.is_new}):
+        for notif in notifications:
+            notif.is_new = False
+        await db_session.commit()
+
+    return tpl_resp
+
 
 @router.get("/object")
 async def admin_object(
@@ -715,7 +773,7 @@ async def admin_object(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
     requested_object = await boxes.get_anybox_object_by_ap_id(db_session, ap_id)
-    if not requested_object:
+    if not requested_object or requested_object.is_deleted:
         raise HTTPException(status_code=404)
 
     replies_tree = await boxes.get_replies_tree(
@@ -736,8 +794,10 @@ async def admin_object(
 async def admin_profile(
     request: Request,
     actor_id: str,
+    cursor: str | None = None,
     db_session: AsyncSession = Depends(get_db_session),
 ) -> templates.TemplateResponse:
+    # TODO: show featured/pinned
     actor = (
         await db_session.execute(
             select(models.Actor).where(models.Actor.ap_id == actor_id)
@@ -748,17 +808,27 @@ async def admin_profile(
 
     actors_metadata = await get_actors_metadata(db_session, [actor])
 
-    inbox_objects = (
-        (
-            await db_session.scalars(
-                select(models.InboxObject)
-                .where(
+    where = [
         models.InboxObject.is_deleted.is_(False),
         models.InboxObject.actor_id == actor.id,
         models.InboxObject.ap_type.in_(
             ["Note", "Article", "Video", "Page", "Announce"]
         ),
+    ]
+    if cursor:
+        decoded_cursor = pagination.decode_cursor(cursor)
+        where.append(models.InboxObject.ap_published_at < decoded_cursor)
+
+    page_size = 20
+    remaining_count = await db_session.scalar(
+        select(func.count(models.InboxObject.id)).where(*where)
     )
+
+    inbox_objects = (
+        (
+            await db_session.scalars(
+                select(models.InboxObject)
+                .where(*where)
                 .options(
                     joinedload(models.InboxObject.relates_to_inbox_object).options(
                         joinedload(models.InboxObject.actor)
@@ -771,12 +841,19 @@ async def admin_profile(
                     joinedload(models.InboxObject.actor),
                 )
                 .order_by(models.InboxObject.ap_published_at.desc())
+                .limit(page_size)
             )
         )
         .unique()
         .all()
     )
 
+    next_cursor = (
+        pagination.encode_cursor(inbox_objects[-1].created_at)
+        if inbox_objects and remaining_count > page_size
+        else None
+    )
+
     return await templates.render_template(
         db_session,
         request,
@@ -785,10 +862,71 @@ async def admin_profile(
             "actors_metadata": actors_metadata,
             "actor": actor,
             "inbox_objects": inbox_objects,
+            "next_cursor": next_cursor,
         },
     )
 
 
+@router.post("/actions/force_delete")
+async def admin_actions_force_delete(
+    request: Request,
+    ap_object_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    ap_object_to_delete = await get_inbox_object_by_ap_id(db_session, ap_object_id)
+    if not ap_object_to_delete:
+        raise ValueError(f"Cannot find {ap_object_id}")
+
+    logger.info(f"Deleting {ap_object_to_delete.ap_type}/{ap_object_to_delete.ap_id}")
+    await boxes._revert_side_effect_for_deleted_object(
+        db_session,
+        None,
+        ap_object_to_delete,
+        None,
+    )
+    ap_object_to_delete.is_deleted = True
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
+@router.post("/actions/force_delete_webmention")
+async def admin_actions_force_delete_webmention(
+    request: Request,
+    webmention_id: int = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    webmention = await boxes.get_webmention_by_id(db_session, webmention_id)
+    if not webmention:
+        raise ValueError(f"Cannot find {webmention_id}")
+    if not webmention.outbox_object:
+        raise ValueError(f"Missing related outbox object for {webmention_id}")
+
+    # TODO: move this
+    logger.info(f"Deleting {webmention_id}")
+    webmention.is_deleted = True
+    await db_session.flush()
+    from app.webmentions import _handle_webmention_side_effects
+
+    await _handle_webmention_side_effects(
+        db_session, webmention, webmention.outbox_object
+    )
+    # Delete related notifications
+    notif_deletion_result = await db_session.execute(
+        delete(models.Notification)
+        .where(models.Notification.webmention_id == webmention.id)
+        .execution_options(synchronize_session=False)
+    )
+    logger.info(
+        f"Deleted {notif_deletion_result.rowcount} notifications"  # type: ignore
+    )
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
 @router.post("/actions/follow")
 async def admin_actions_follow(
     request: Request,
@@ -810,10 +948,7 @@ async def admin_actions_block(
     csrf_check: None = Depends(verify_csrf_token),
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
-    logger.info(f"Blocking {ap_actor_id}")
-    actor = await fetch_actor(db_session, ap_actor_id)
-    actor.is_blocked = True
-    await db_session.commit()
+    await send_block(db_session, ap_actor_id)
     return RedirectResponse(redirect_url, status_code=302)
 
 
@@ -826,8 +961,34 @@ async def admin_actions_unblock(
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
     logger.info(f"Unblocking {ap_actor_id}")
+    await send_unblock(db_session, ap_actor_id)
+    return RedirectResponse(redirect_url, status_code=302)
+
+
+@router.post("/actions/hide_announces")
+async def admin_actions_hide_announces(
+    request: Request,
+    ap_actor_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
     actor = await fetch_actor(db_session, ap_actor_id)
-    actor.is_blocked = False
+    actor.are_announces_hidden_from_stream = True
+    await db_session.commit()
+    return RedirectResponse(redirect_url, status_code=302)
+
+
+@router.post("/actions/show_announces")
+async def admin_actions_show_announces(
+    request: Request,
+    ap_actor_id: str = Form(),
+    redirect_url: str = Form(),
+    csrf_check: None = Depends(verify_csrf_token),
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse:
+    actor = await fetch_actor(db_session, ap_actor_id)
+    actor.are_announces_hidden_from_stream = False
     await db_session.commit()
     return RedirectResponse(redirect_url, status_code=302)
 
@@ -974,7 +1135,7 @@ async def admin_actions_unpin(
 async def admin_actions_new(
     request: Request,
     files: list[UploadFile] = [],
-    content: str = Form(),
+    content: str | None = Form(None),
     redirect_url: str = Form(),
     in_reply_to: str | None = Form(None),
     content_warning: str | None = Form(None),
@@ -985,6 +1146,19 @@ async def admin_actions_new(
     csrf_check: None = Depends(verify_csrf_token),
     db_session: AsyncSession = Depends(get_db_session),
 ) -> RedirectResponse:
+    if not content and not content_warning:
+        raise HTTPException(status_code=422, detail="Error: object must have a content")
+
+    # Do like Mastodon, if there's only a CW with no content and some attachments,
+    # swap the CW and the content
+    if not content and content_warning and len(files) >= 1:
+        content = content_warning
+        is_sensitive = True
+        content_warning = None
+
+    if not content:
+        raise HTTPException(status_code=422, detail="Error: objec must have a content")
+
     # XXX: for some reason, no files restuls in an empty single file
     uploads = []
     raw_form_data = await request.form()
@@ -1011,7 +1185,7 @@ async def admin_actions_new(
     elif name:
         ap_type = "Article"
 
-    public_id = await boxes.send_create(
+    public_id, _ = await boxes.send_create(
         db_session,
         ap_type=ap_type,
         source=content,
@@ -1054,7 +1228,10 @@ async def admin_actions_vote(
 async def login(
     request: Request,
     db_session: AsyncSession = Depends(get_db_session),
-) -> templates.TemplateResponse:
+) -> templates.TemplateResponse | RedirectResponse:
+    if is_current_user_admin(request):
+        return RedirectResponse(request.url_for("admin_stream"), status_code=302)
+
     return await templates.render_template(
         db_session,
         request,
@@ -1072,11 +1249,25 @@ async def login_validation(
     password: str = Form(),
     redirect: str | None = Form(None),
     csrf_check: None = Depends(verify_csrf_token),
-) -> RedirectResponse:
+    db_session: AsyncSession = Depends(get_db_session),
+) -> RedirectResponse | templates.TemplateResponse:
     if not verify_password(password):
-        raise HTTPException(status_code=401)
+        logger.warning("Invalid password")
+        return await templates.render_template(
+            db_session,
+            request,
+            "login.html",
+            {
+                "error": "Invalid password",
+                "csrf_token": generate_csrf_token(),
+                "redirect": request.query_params.get("redirect", ""),
+            },
+            status_code=403,
+        )
 
-    resp = RedirectResponse(redirect or "/admin/stream", status_code=302)
+    resp = RedirectResponse(
+        redirect or request.url_for("admin_stream"), status_code=302
+    )
     resp.set_cookie("session", session_serializer.dumps({"is_logged_in": True}))  # type: ignore  # noqa: E501
 
     return resp

app/ap_object.py

@@ -1,16 +1,18 @@
 import hashlib
+import mimetypes
 from datetime import datetime
 from functools import cached_property
 from typing import Any
 
 import pydantic
 from bs4 import BeautifulSoup  # type: ignore
-from markdown import markdown
+from mistletoe import markdown  # type: ignore
 
 from app import activitypub as ap
 from app.actor import LOCAL_ACTOR
 from app.actor import Actor
 from app.actor import RemoteActor
+from app.config import ID
 from app.media import proxied_media_url
 from app.utils.datetime import now
 from app.utils.datetime import parse_isoformat
@@ -95,6 +97,9 @@ class Object:
     def attachments(self) -> list["Attachment"]:
         attachments = []
         for obj in ap.as_list(self.ap_object.get("attachment", [])):
+            if obj.get("type") == "PropertyValue":
+                continue
+
             if obj.get("type") == "Link":
                 attachments.append(
                     Attachment.parse_obj(
@@ -155,7 +160,7 @@ class Object:
     @cached_property
     def url(self) -> str | None:
         obj_url = self.ap_object.get("url")
-        if isinstance(obj_url, str):
+        if isinstance(obj_url, str) and obj_url:
             return obj_url
         elif obj_url:
             for u in ap.as_list(obj_url):
@@ -175,7 +180,7 @@ class Object:
 
         # PeerTube returns the content as markdown
         if self.ap_object.get("mediaType") == "text/markdown":
-            content = markdown(content, extensions=["mdx_linkify"])
+            content = markdown(content)
 
         return content
 
@@ -208,6 +213,22 @@ class Object:
     def in_reply_to(self) -> str | None:
         return self.ap_object.get("inReplyTo")
 
+    @property
+    def is_local_reply(self) -> bool:
+        if not self.in_reply_to:
+            return False
+
+        return bool(
+            self.in_reply_to.startswith(ID) and self.content  # Hide votes from Question
+        )
+
+    @property
+    def is_in_reply_to_from_inbox(self) -> bool | None:
+        if not self.in_reply_to:
+            return None
+
+        return not self.in_reply_to.startswith(LOCAL_ACTOR.ap_id)
+
     @property
     def has_ld_signature(self) -> bool:
         return bool(self.ap_object.get("signature"))
@@ -269,6 +290,20 @@ class Attachment(BaseModel):
     proxied_url: str | None = None
     resized_url: str | None = None
 
+    width: int | None = None
+    height: int | None = None
+
+    @property
+    def mimetype(self) -> str:
+        mimetype = self.media_type
+        if not mimetype:
+            mimetype, _ = mimetypes.guess_type(self.url)
+
+        if not mimetype:
+            return "unknown"
+
+        return mimetype.split("/")[-1]
+
 
 class RemoteObject(Object):
     def __init__(self, raw_object: ap.RawObject, actor: Actor):

app/config.py

@@ -1,4 +1,5 @@
 import hashlib
+import hmac
 import os
 import secrets
 from pathlib import Path
@@ -12,8 +13,13 @@ from fastapi import HTTPException
 from fastapi import Request
 from itsdangerous import URLSafeTimedSerializer
 from loguru import logger
+from mistletoe import markdown  # type: ignore
 
+from app.customization import _CUSTOM_ROUTES
+from app.customization import _StreamVisibilityCallback
+from app.customization import default_stream_visibility_callback
 from app.utils.emoji import _load_emojis
+from app.utils.version import get_version_commit
 
 ROOT_DIR = Path().parent.resolve()
 
@@ -24,7 +30,7 @@ VERSION_COMMIT = "dev"
 try:
     from app._version import VERSION_COMMIT  # type: ignore
 except ImportError:
-    pass
+    VERSION_COMMIT = get_version_commit()
 
 # Force reloading cache when the CSS is updated
 CSS_HASH = "none"
@@ -34,6 +40,34 @@ try:
 except FileNotFoundError:
     pass
 
+# Force reloading cache when the JS is changed
+JS_HASH = "none"
+try:
+    # To keep things simple, we keep a single hash for the 2 files
+    dat = b""
+    for j in [
+        ROOT_DIR / "app" / "static" / "common.js",
+        ROOT_DIR / "app" / "static" / "common-admin.js",
+        ROOT_DIR / "app" / "static" / "new.js",
+    ]:
+        dat += j.read_bytes()
+    JS_HASH = hashlib.md5(dat, usedforsecurity=False).hexdigest()
+except FileNotFoundError:
+    pass
+
+MOVED_TO_FILE = ROOT_DIR / "data" / "moved_to.dat"
+
+
+def _get_moved_to() -> str | None:
+    if not MOVED_TO_FILE.exists():
+        return None
+
+    return MOVED_TO_FILE.read_text()
+
+
+def set_moved_to(moved_to: str) -> None:
+    MOVED_TO_FILE.write_text(moved_to)
+
 
 VERSION = f"2.0.0+{VERSION_COMMIT}"
 USER_AGENT = f"microblogpub/{VERSION}"
@@ -62,7 +96,8 @@ class Config(pydantic.BaseModel):
     name: str
     summary: str
     https: bool
-    icon_url: str
+    icon_url: str | None = None
+    image_url: str | None = None
     secret: str
     debug: bool = False
     trusted_hosts: list[str] = ["127.0.0.1"]
@@ -71,13 +106,31 @@ class Config(pydantic.BaseModel):
     metadata: list[_ProfileMetadata] | None = None
     code_highlighting_theme = "friendly_grayscale"
     blocked_servers: list[_BlockedServer] = []
+    custom_footer: str | None = None
+    emoji: str | None = None
+    also_known_as: str | None = None
+
+    hides_followers: bool = False
+    hides_following: bool = False
 
     inbox_retention_days: int = 15
 
+    custom_content_security_policy: str | None = None
+
+    webfinger_domain: str | None = None
+
     # Config items to make tests easier
     sqlalchemy_database: str | None = None
     key_path: str | None = None
 
+    session_timeout: int = 3600 * 24 * 3  # in seconds, 3 days by default
+    csrf_token_exp: int = 3600
+
+    disabled_notifications: list[str] = []
+
+    # Only set when the app is served on a non-root path
+    id: str | None = None
+
 
 def load_config() -> Config:
     try:
@@ -112,15 +165,34 @@ CONFIG = load_config()
 DOMAIN = CONFIG.domain
 _SCHEME = "https" if CONFIG.https else "http"
 ID = f"{_SCHEME}://{DOMAIN}"
+
+# When running the app on a path, the ID maybe set by the config, but in this
+# case, a valid webfinger must be served on the root domain
+if CONFIG.id:
+    ID = CONFIG.id
 USERNAME = CONFIG.username
+
+# Allow to use @handle@webfinger-domain.tld while hosting the server at domain.tld
+WEBFINGER_DOMAIN = CONFIG.webfinger_domain or DOMAIN
+
 MANUALLY_APPROVES_FOLLOWERS = CONFIG.manually_approves_followers
+HIDES_FOLLOWERS = CONFIG.hides_followers
+HIDES_FOLLOWING = CONFIG.hides_following
 PRIVACY_REPLACE = None
 if CONFIG.privacy_replace:
     PRIVACY_REPLACE = {pr.domain: pr.replace_by for pr in CONFIG.privacy_replace}
 
 BLOCKED_SERVERS = {blocked_server.hostname for blocked_server in CONFIG.blocked_servers}
+ALSO_KNOWN_AS = CONFIG.also_known_as
+CUSTOM_CONTENT_SECURITY_POLICY = CONFIG.custom_content_security_policy
 
 INBOX_RETENTION_DAYS = CONFIG.inbox_retention_days
+SESSION_TIMEOUT = CONFIG.session_timeout
+CUSTOM_FOOTER = (
+    markdown(CONFIG.custom_footer.replace("{version}", VERSION))
+    if CONFIG.custom_footer
+    else None
+)
 
 BASE_URL = ID
 DEBUG = CONFIG.debug
@@ -130,13 +202,45 @@ KEY_PATH = (
     (ROOT_DIR / CONFIG.key_path) if CONFIG.key_path else ROOT_DIR / "data" / "key.pem"
 )
 EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾"
+if CONFIG.emoji:
+    EMOJIS = CONFIG.emoji
+
 # Emoji template for the FE
-EMOJI_TPL = '<img src="/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
+EMOJI_TPL = (
+    '<img src="{base_url}/static/twemoji/{filename}.svg" alt="{raw}" class="emoji">'
+)
 
 _load_emojis(ROOT_DIR, BASE_URL)
 
 CODE_HIGHLIGHTING_THEME = CONFIG.code_highlighting_theme
 
+MOVED_TO = _get_moved_to()
+
+
+_NavBarItem = tuple[str, str]
+
+
+class NavBarItems:
+    EXTRA_NAVBAR_ITEMS: list[_NavBarItem] = []
+    INDEX_NAVBAR_ITEM: _NavBarItem | None = None
+    NOTES_PATH = "/"
+
+
+def load_custom_routes() -> None:
+    try:
+        from data import custom_routes  # type: ignore  # noqa: F401
+    except ImportError:
+        pass
+
+    for path, custom_handler in _CUSTOM_ROUTES.items():
+        # If a handler wants to replace the root, move the index to /notes
+        if path == "/":
+            NavBarItems.NOTES_PATH = "/notes"
+            NavBarItems.INDEX_NAVBAR_ITEM = (path, custom_handler.title)
+        else:
+            if custom_handler.show_in_navbar:
+                NavBarItems.EXTRA_NAVBAR_ITEMS.append((path, custom_handler.title))
+
 
 session_serializer = URLSafeTimedSerializer(
     CONFIG.secret,
@@ -152,10 +256,34 @@ def generate_csrf_token() -> str:
     return csrf_serializer.dumps(secrets.token_hex(16))  # type: ignore
 
 
-def verify_csrf_token(csrf_token: str = Form()) -> None:
+def verify_csrf_token(
+    csrf_token: str = Form(),
+    redirect_url: str | None = Form(None),
+) -> None:
+    please_try_again = "please try again"
+    if redirect_url:
+        please_try_again = f'<a href="{redirect_url}">please try again</a>'
     try:
-        csrf_serializer.loads(csrf_token, max_age=1800)
+        csrf_serializer.loads(csrf_token, max_age=CONFIG.csrf_token_exp)
     except (itsdangerous.BadData, itsdangerous.SignatureExpired):
         logger.exception("Failed to verify CSRF token")
-        raise HTTPException(status_code=403, detail="CSRF error")
+        raise HTTPException(
+            status_code=403,
+            detail=f"The security token has expired, {please_try_again}",
+        )
     return None
+
+
+def hmac_sha256() -> hmac.HMAC:
+    return hmac.new(CONFIG.secret.encode(), digestmod=hashlib.sha256)
+
+
+stream_visibility_callback: _StreamVisibilityCallback
+try:
+    from data.stream import (  # type: ignore  # noqa: F401, E501
+        custom_stream_visibility_callback,
+    )
+
+    stream_visibility_callback = custom_stream_visibility_callback
+except ImportError:
+    stream_visibility_callback = default_stream_visibility_callback

app/customization.py

@@ -0,0 +1,155 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import Callable
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Request
+from loguru import logger
+from starlette.responses import JSONResponse
+
+if TYPE_CHECKING:
+    from app.ap_object import RemoteObject
+
+
+_DATA_DIR = Path().parent.resolve() / "data"
+_Handler = Callable[..., Any]
+
+
+class HTMLPage:
+    def __init__(
+        self,
+        title: str,
+        html_file: str,
+        show_in_navbar: bool,
+    ) -> None:
+        self.title = title
+        self.html_file = _DATA_DIR / html_file
+        self.show_in_navbar = show_in_navbar
+
+
+class RawHandler:
+    def __init__(
+        self,
+        title: str,
+        handler: Any,
+        show_in_navbar: bool,
+    ) -> None:
+        self.title = title
+        self.handler = handler
+        self.show_in_navbar = show_in_navbar
+
+
+_CUSTOM_ROUTES: dict[str, HTMLPage | RawHandler] = {}
+
+
+def register_html_page(
+    path: str,
+    *,
+    title: str,
+    html_file: str,
+    show_in_navbar: bool = True,
+) -> None:
+    if path in _CUSTOM_ROUTES:
+        raise ValueError(f"{path} is already registered")
+
+    _CUSTOM_ROUTES[path] = HTMLPage(title, html_file, show_in_navbar)
+
+
+def register_raw_handler(
+    path: str,
+    *,
+    title: str,
+    handler: _Handler,
+    show_in_navbar: bool = True,
+) -> None:
+    if path in _CUSTOM_ROUTES:
+        raise ValueError(f"{path} is already registered")
+
+    _CUSTOM_ROUTES[path] = RawHandler(title, handler, show_in_navbar)
+
+
+class ActivityPubResponse(JSONResponse):
+    media_type = "application/activity+json"
+
+
+def _custom_page_handler(path: str, html_page: HTMLPage) -> Any:
+    from app import templates
+    from app.actor import LOCAL_ACTOR
+    from app.config import is_activitypub_requested
+    from app.database import AsyncSession
+    from app.database import get_db_session
+
+    async def _handler(
+        request: Request,
+        db_session: AsyncSession = Depends(get_db_session),
+    ) -> templates.TemplateResponse | ActivityPubResponse:
+        if path == "/" and is_activitypub_requested(request):
+            return ActivityPubResponse(LOCAL_ACTOR.ap_actor)
+
+        return await templates.render_template(
+            db_session,
+            request,
+            "custom_page.html",
+            {
+                "page_content": html_page.html_file.read_text(),
+                "title": html_page.title,
+            },
+        )
+
+    return _handler
+
+
+def get_custom_router() -> APIRouter | None:
+    if not _CUSTOM_ROUTES:
+        return None
+
+    router = APIRouter()
+
+    for path, handler in _CUSTOM_ROUTES.items():
+        if isinstance(handler, HTMLPage):
+            router.add_api_route(
+                path, _custom_page_handler(path, handler), methods=["GET"]
+            )
+        else:
+            router.add_api_route(path, handler.handler)
+
+    return router
+
+
+@dataclass
+class ObjectInfo:
+    # Is it a reply?
+    is_reply: bool
+
+    # Is it a reply to an outbox object
+    is_local_reply: bool
+
+    # Is the object mentioning the local actor
+    is_mention: bool
+
+    # Is it from someone the local actor is following
+    is_from_following: bool
+
+    # List of hashtags, e.g. #microblogpub
+    hashtags: list[str]
+
+    # @dev@microblog.pub
+    actor_handle: str
+
+    remote_object: "RemoteObject"
+
+
+_StreamVisibilityCallback = Callable[[ObjectInfo], bool]
+
+
+def default_stream_visibility_callback(object_info: ObjectInfo) -> bool:
+    result = (
+        (not object_info.is_reply and object_info.is_from_following)
+        or object_info.is_mention
+        or object_info.is_local_reply
+    )
+    logger.info(f"{object_info=}/{result=}")
+    return result

app/database.py

@@ -9,6 +9,7 @@ from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
 from app.config import DB_PATH
+from app.config import DEBUG
 from app.config import SQLALCHEMY_DATABASE_URL
 
 engine = create_engine(
@@ -18,7 +19,7 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 DATABASE_URL = f"sqlite+aiosqlite:///{DB_PATH}"
 async_engine = create_async_engine(
-    DATABASE_URL, future=True, echo=False, connect_args={"timeout": 15}
+    DATABASE_URL, future=True, echo=DEBUG, connect_args={"timeout": 15}
 )
 async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False)
 

app/httpsig.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import json
 import typing
 from dataclasses import dataclass
 from datetime import datetime
@@ -22,12 +23,12 @@ from sqlalchemy import select
 
 from app import activitypub as ap
 from app import config
-from app.config import BLOCKED_SERVERS
 from app.config import KEY_PATH
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.key import Key
 from app.utils.datetime import now
+from app.utils.url import is_hostname_blocked
 
 _KEY_CACHE: MutableMapping[str, Key] = LFUCache(256)
 
@@ -88,8 +89,12 @@ def _body_digest(body: bytes) -> str:
     return "SHA-256=" + base64.b64encode(h.digest()).decode("utf-8")
 
 
-async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
-    if cached_key := _KEY_CACHE.get(key_id):
+async def _get_public_key(
+    db_session: AsyncSession,
+    key_id: str,
+    should_skip_cache: bool = False,
+) -> Key:
+    if not should_skip_cache and (cached_key := _KEY_CACHE.get(key_id)):
         logger.info(f"Key {key_id} found in cache")
         return cached_key
 
@@ -101,6 +106,7 @@ async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
             select(models.Actor).where(models.Actor.ap_id == key_id.split("#")[0])
         )
     ).one_or_none()
+    if not should_skip_cache:
         if existing_actor and existing_actor.public_key_id == key_id:
             k = Key(existing_actor.ap_id, key_id)
             k.load_pub(existing_actor.public_key_as_pem)
@@ -110,16 +116,15 @@ async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
 
     # Fetch it
     from app import activitypub as ap
+    from app.actor import RemoteActor
+    from app.actor import update_actor_if_needed
 
     # Without signing the request as if it's the first contact, the 2 servers
     # might race to fetch each other key
     try:
         actor = await ap.fetch(key_id, disable_httpsig=True)
-    except httpx.HTTPStatusError as http_err:
-        if http_err.response.status_code in [401, 403]:
+    except ap.ObjectUnavailableError:
         actor = await ap.fetch(key_id, disable_httpsig=False)
-        else:
-            raise
 
     if actor["type"] == "Key":
         # The Key is not embedded in the Person
@@ -130,11 +135,18 @@ async def _get_public_key(db_session: AsyncSession, key_id: str) -> Key:
         k.load_pub(actor["publicKey"]["publicKeyPem"])
 
     # Ensure the right key was fetch
+    # TODO: some server have the key ID `http://` but fetching it return `https`
     if key_id not in [k.key_id(), k.owner]:
         raise ValueError(
             f"failed to fetch requested key {key_id}: got {actor['publicKey']}"
         )
 
+    if should_skip_cache and actor["type"] != "Key" and existing_actor:
+        # We had to skip the cache, which means the actor key probably changed
+        # and we want to update our cached version
+        await update_actor_if_needed(db_session, existing_actor, RemoteActor(actor))
+        await db_session.commit()
+
     _KEY_CACHE[key_id] = k
     return k
 
@@ -172,7 +184,7 @@ async def httpsig_checker(
         )
 
     server = urlparse(key_id).hostname
-    if server in BLOCKED_SERVERS:
+    if is_hostname_blocked(server):
         return HTTPSigInfo(
             has_valid_signature=False,
             server=server,
@@ -187,6 +199,32 @@ async def httpsig_checker(
             server=server,
         )
 
+    # Try to drop Delete activity spams early on, this prevent making an extra
+    # HTTP requests trying to fetch an unavailable actor to verify the HTTP sig
+    try:
+        if request.method == "POST" and request.url.path.endswith("/inbox"):
+            from app import models  # TODO: solve this circular import
+
+            activity = json.loads(body)
+            actor_id = ap.get_id(activity["actor"])
+            if (
+                ap.as_list(activity["type"])[0] == "Delete"
+                and actor_id == ap.get_id(activity["object"])
+                and not (
+                    await db_session.scalars(
+                        select(models.Actor).where(
+                            models.Actor.ap_id == actor_id,
+                        )
+                    )
+                ).one_or_none()
+            ):
+                logger.info(f"Dropping Delete activity early for {body=}")
+                raise fastapi.HTTPException(status_code=202)
+    except fastapi.HTTPException as http_exc:
+        raise http_exc
+    except Exception:
+        logger.exception("Failed to check for Delete spam")
+
     # logger.debug(f"hsig={hsig}")
     signed_string, signature_date = _build_signed_string(
         hsig["headers"],
@@ -215,10 +253,23 @@ async def httpsig_checker(
         logger.exception(f'Failed to fetch HTTP sig key {hsig["keyId"]}')
         return HTTPSigInfo(has_valid_signature=False)
 
-    httpsig_info = HTTPSigInfo(
-        has_valid_signature=_verify_h(
+    has_valid_signature = _verify_h(
         signed_string, base64.b64decode(hsig["signature"]), k.pubkey
-        ),
+    )
+
+    # If the signature is not valid, we may have to update the cached actor
+    if not has_valid_signature:
+        logger.info("Invalid signature, trying to refresh actor")
+        try:
+            k = await _get_public_key(db_session, hsig["keyId"], should_skip_cache=True)
+            has_valid_signature = _verify_h(
+                signed_string, base64.b64decode(hsig["signature"]), k.pubkey
+            )
+        except Exception:
+            logger.exception("Failed to refresh actor")
+
+    httpsig_info = HTTPSigInfo(
+        has_valid_signature=has_valid_signature,
         signed_by_ap_actor_id=k.owner,
         server=server,
     )

app/incoming_activities.py

@@ -3,7 +3,6 @@ import traceback
 from datetime import datetime
 from datetime import timedelta
 
-import httpx
 from loguru import logger
 from sqlalchemy import func
 from sqlalchemy import select
@@ -26,7 +25,7 @@ async def new_ap_incoming_activity(
     raw_object: ap.RawObject,
 ) -> models.IncomingActivity | None:
     ap_id: str
-    if "id" not in raw_object:
+    if "id" not in raw_object or ap.as_list(raw_object["type"])[0] in ap.ACTOR_TYPES:
         if "@context" not in raw_object:
             logger.warning(f"Dropping invalid object: {raw_object}")
             return None
@@ -61,7 +60,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:
@@ -108,22 +107,29 @@ async def process_next_incoming_activity(
 
     next_activity.tries = next_activity.tries + 1
     next_activity.last_try = now()
+    await db_session.commit()
 
     if next_activity.ap_object and next_activity.sent_by_ap_actor_id:
         try:
             async with db_session.begin_nested():
-                await save_to_inbox(
+                await asyncio.wait_for(
+                    save_to_inbox(
                         db_session,
                         next_activity.ap_object,
                         next_activity.sent_by_ap_actor_id,
+                    ),
+                    timeout=60,
                 )
-        except httpx.TimeoutException as exc:
-            url = exc._request.url if exc._request else None
-            logger.error(f"Failed, HTTP timeout when fetching {url}")
+        except asyncio.exceptions.TimeoutError:
+            logger.error("Activity took too long to process")
+            await db_session.rollback()
+            await db_session.refresh(next_activity)
             next_activity.error = traceback.format_exc()
             _set_next_try(next_activity)
         except Exception:
             logger.exception("Failed")
+            await db_session.rollback()
+            await db_session.refresh(next_activity)
             next_activity.error = traceback.format_exc()
             _set_next_try(next_activity)
         else:

app/indieauth.py

@@ -10,9 +10,12 @@ from fastapi import Form
 from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
-from fastapi.responses import RedirectResponse
+from fastapi.security import HTTPBasic
+from fastapi.security import HTTPBasicCredentials
 from loguru import logger
+from pydantic import BaseModel
 from sqlalchemy import select
+from sqlalchemy.orm import joinedload
 
 from app import config
 from app import models
@@ -21,9 +24,12 @@ from app.admin import user_session_or_redirect
 from app.config import verify_csrf_token
 from app.database import AsyncSession
 from app.database import get_db_session
+from app.redirect import redirect
 from app.utils import indieauth
 from app.utils.datetime import now
 
+basic_auth = HTTPBasic()
+
 router = APIRouter()
 
 
@@ -38,9 +44,55 @@ async def well_known_authorization_server(
         "code_challenge_methods_supported": ["S256"],
         "revocation_endpoint": request.url_for("indieauth_revocation_endpoint"),
         "revocation_endpoint_auth_methods_supported": ["none"],
+        "registration_endpoint": request.url_for("oauth_registration_endpoint"),
+        "introspection_endpoint": request.url_for("oauth_introspection_endpoint"),
     }
 
 
+class OAuthRegisterClientRequest(BaseModel):
+    client_name: str
+    redirect_uris: list[str] | str
+
+    client_uri: str | None = None
+    logo_uri: str | None = None
+    scope: str | None = None
+
+
+@router.post("/oauth/register")
+async def oauth_registration_endpoint(
+    register_client_request: OAuthRegisterClientRequest,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> JSONResponse:
+    """Implements OAuth 2.0 Dynamic Registration."""
+
+    client = models.OAuthClient(
+        client_name=register_client_request.client_name,
+        redirect_uris=[register_client_request.redirect_uris]
+        if isinstance(register_client_request.redirect_uris, str)
+        else register_client_request.redirect_uris,
+        client_uri=register_client_request.client_uri,
+        logo_uri=register_client_request.logo_uri,
+        scope=register_client_request.scope,
+        client_id=secrets.token_hex(16),
+        client_secret=secrets.token_hex(32),
+    )
+
+    db_session.add(client)
+    await db_session.commit()
+
+    return JSONResponse(
+        content={
+            **register_client_request.dict(),
+            "client_id_issued_at": int(client.created_at.timestamp()),  # type: ignore
+            "grant_types": ["authorization_code", "refresh_token"],
+            "client_secret_expires_at": 0,
+            "client_id": client.client_id,
+            "client_secret": client.client_secret,
+        },
+        status_code=201,
+    )
+
+
 @router.get("/auth")
 async def indieauth_authorization_endpoint(
     request: Request,
@@ -56,12 +108,29 @@ async def indieauth_authorization_endpoint(
     code_challenge = request.query_params.get("code_challenge", "")
     code_challenge_method = request.query_params.get("code_challenge_method", "")
 
+    # Check if the authorization request is coming from an OAuth client
+    registered_client = (
+        await db_session.scalars(
+            select(models.OAuthClient).where(
+                models.OAuthClient.client_id == client_id,
+            )
+        )
+    ).one_or_none()
+    if registered_client:
+        client = {
+            "name": registered_client.client_name,
+            "logo": registered_client.logo_uri,
+            "url": registered_client.client_uri,
+        }
+    else:
+        client = await indieauth.get_client_id_data(client_id)  # type: ignore
+
     return await templates.render_template(
         db_session,
         request,
         "indieauth_flow.html",
         dict(
-            client=await indieauth.get_client_id_data(client_id),
+            client=client,
             scopes=scope,
             redirect_uri=redirect_uri,
             state=state,
@@ -80,7 +149,7 @@ async def indieauth_flow(
     db_session: AsyncSession = Depends(get_db_session),
     csrf_check: None = Depends(verify_csrf_token),
     _: None = Depends(user_session_or_redirect),
-) -> RedirectResponse:
+) -> templates.TemplateResponse:
     form_data = await request.form()
     logger.info(f"{form_data=}")
 
@@ -114,9 +183,8 @@ async def indieauth_flow(
     db_session.add(auth_request)
     await db_session.commit()
 
-    return RedirectResponse(
-        redirect_uri + f"?code={code}&state={state}&iss={iss}",
-        status_code=302,
+    return await redirect(
+        request, db_session, redirect_uri + f"?code={code}&state={state}&iss={iss}"
     )
 
 
@@ -207,17 +275,17 @@ async def indieauth_token_endpoint(
     form_data = await request.form()
     logger.info(f"{form_data=}")
     grant_type = form_data.get("grant_type", "authorization_code")
-    if grant_type != "authorization_code":
+    if grant_type not in ["authorization_code", "refresh_token"]:
         raise ValueError(f"Invalid grant_type {grant_type}")
 
-    code = form_data["code"]
-
     # These must match the params from the first request
     client_id = form_data["client_id"]
-    redirect_uri = form_data["redirect_uri"]
-    # code_verifier is optional for backward compat
     code_verifier = form_data.get("code_verifier")
 
+    if grant_type == "authorization_code":
+        code = form_data["code"]
+        redirect_uri = form_data["redirect_uri"]
+        # code_verifier is optional for backward compat
         is_code_valid, auth_code_request = await _check_auth_code(
             db_session,
             code=code,
@@ -231,12 +299,38 @@ async def indieauth_token_endpoint(
                 status_code=400,
             )
 
+    elif grant_type == "refresh_token":
+        refresh_token = form_data["refresh_token"]
+        access_token = (
+            await db_session.scalars(
+                select(models.IndieAuthAccessToken)
+                .where(
+                    models.IndieAuthAccessToken.refresh_token == refresh_token,
+                    models.IndieAuthAccessToken.was_refreshed.is_(False),
+                )
+                .options(
+                    joinedload(
+                        models.IndieAuthAccessToken.indieauth_authorization_request
+                    )
+                )
+            )
+        ).one_or_none()
+        if not access_token:
+            raise ValueError("invalid refresh token")
+
+        if access_token.indieauth_authorization_request.client_id != client_id:
+            raise ValueError("invalid client ID")
+
+        auth_code_request = access_token.indieauth_authorization_request
+        access_token.was_refreshed = True
+
     if not auth_code_request:
         raise ValueError("Should never happen")
 
     access_token = models.IndieAuthAccessToken(
         indieauth_authorization_request_id=auth_code_request.id,
         access_token=secrets.token_urlsafe(32),
+        refresh_token=secrets.token_urlsafe(32),
         expires_in=3600,
         scope=auth_code_request.scope,
     )
@@ -246,6 +340,7 @@ async def indieauth_token_endpoint(
     return JSONResponse(
         content={
             "access_token": access_token.access_token,
+            "refresh_token": access_token.refresh_token,
             "token_type": "Bearer",
             "scope": auth_code_request.scope,
             "me": config.ID + "/",
@@ -261,8 +356,10 @@ async def _check_access_token(
 ) -> tuple[bool, models.IndieAuthAccessToken | None]:
     access_token_info = (
         await db_session.scalars(
-            select(models.IndieAuthAccessToken).where(
-                models.IndieAuthAccessToken.access_token == token
+            select(models.IndieAuthAccessToken)
+            .where(models.IndieAuthAccessToken.access_token == token)
+            .options(
+                joinedload(models.IndieAuthAccessToken.indieauth_authorization_request)
             )
         )
     ).one_or_none()
@@ -276,7 +373,7 @@ async def _check_access_token(
     if now() > access_token_info.created_at.replace(tzinfo=timezone.utc) + timedelta(
         seconds=access_token_info.expires_in
     ):
-        logger.info("Access token is expired")
+        logger.info("Access token has expired")
         return False, None
 
     return True, access_token_info
@@ -285,6 +382,9 @@ async def _check_access_token(
 @dataclass(frozen=True)
 class AccessTokenInfo:
     scopes: list[str]
+    client_id: str | None
+    access_token: str
+    exp: int
 
 
 async def verify_access_token(
@@ -311,9 +411,71 @@ async def verify_access_token(
 
     return AccessTokenInfo(
         scopes=access_token.scope.split(),
+        client_id=(
+            access_token.indieauth_authorization_request.client_id
+            if access_token.indieauth_authorization_request
+            else None
+        ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
     )
 
 
+async def check_access_token(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> AccessTokenInfo | None:
+    token = request.headers.get("Authorization", "").removeprefix("Bearer ")
+    if not token:
+        return None
+
+    is_token_valid, access_token = await _check_access_token(db_session, token)
+    if not is_token_valid:
+        return None
+
+    if not access_token or not access_token.scope:
+        raise ValueError("Should never happen")
+
+    access_token_info = AccessTokenInfo(
+        scopes=access_token.scope.split(),
+        client_id=(
+            access_token.indieauth_authorization_request.client_id
+            if access_token.indieauth_authorization_request
+            else None
+        ),
+        access_token=access_token.access_token,
+        exp=int(
+            (
+                access_token.created_at.replace(tzinfo=timezone.utc)
+                + timedelta(seconds=access_token.expires_in)
+            ).timestamp()
+        ),
+    )
+
+    logger.info(
+        "Authenticated with access token from client_id="
+        f"{access_token_info.client_id} scopes={access_token.scope}"
+    )
+
+    return access_token_info
+
+
+async def enforce_access_token(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> AccessTokenInfo:
+    maybe_access_token_info = await check_access_token(request, db_session)
+    if not maybe_access_token_info:
+        raise HTTPException(status_code=401, detail="access token required")
+
+    return maybe_access_token_info
+
+
 @router.post("/revoke_token")
 async def indieauth_revocation_endpoint(
     request: Request,
@@ -333,3 +495,58 @@ async def indieauth_revocation_endpoint(
         content={},
         status_code=200,
     )
+
+
+@router.post("/token_introspection")
+async def oauth_introspection_endpoint(
+    request: Request,
+    credentials: HTTPBasicCredentials = Depends(basic_auth),
+    db_session: AsyncSession = Depends(get_db_session),
+    token: str = Form(),
+) -> JSONResponse:
+    registered_client = (
+        await db_session.scalars(
+            select(models.OAuthClient).where(
+                models.OAuthClient.client_id == credentials.username,
+                models.OAuthClient.client_secret == credentials.password,
+            )
+        )
+    ).one_or_none()
+    if not registered_client:
+        raise HTTPException(status_code=401, detail="unauthenticated")
+
+    access_token = (
+        await db_session.scalars(
+            select(models.IndieAuthAccessToken)
+            .where(models.IndieAuthAccessToken.access_token == token)
+            .join(
+                models.IndieAuthAuthorizationRequest,
+                models.IndieAuthAccessToken.indieauth_authorization_request_id
+                == models.IndieAuthAuthorizationRequest.id,
+            )
+            .where(
+                models.IndieAuthAuthorizationRequest.client_id == credentials.username
+            )
+        )
+    ).one_or_none()
+    if not access_token:
+        return JSONResponse(content={"active": False})
+
+    is_token_valid, _ = await _check_access_token(db_session, token)
+    if not is_token_valid:
+        return JSONResponse(content={"active": False})
+
+    return JSONResponse(
+        content={
+            "active": True,
+            "client_id": credentials.username,
+            "scope": access_token.scope,
+            "exp": int(
+                (
+                    access_token.created_at.replace(tzinfo=timezone.utc)
+                    + timedelta(seconds=access_token.expires_in)
+                ).timestamp()
+            ),
+        },
+        status_code=200,
+    )

app/ldsig.py

@@ -23,6 +23,13 @@ requests_loader = pyld.documentloader.requests.requests_document_loader()
 def _loader(url, options={}):
     # See https://github.com/digitalbazaar/pyld/issues/133
     options["headers"]["Accept"] = "application/ld+json"
+
+    # XXX: temp fix/hack is it seems to be down for now
+    if url == "https://w3id.org/identity/v1":
+        url = (
+            "https://raw.githubusercontent.com/web-payments/web-payments.org"
+            "/master/contexts/identity-v1.jsonld"
+        )
     return requests_loader(url, options)
 
 
@@ -34,7 +41,7 @@ def _options_hash(doc: ap.RawObject) -> str:
     for k in ["type", "id", "signatureValue"]:
         if k in doc:
             del doc[k]
-    doc["@context"] = "https://w3id.org/identity/v1"
+    doc["@context"] = "https://w3id.org/security/v1"
     normalized = jsonld.normalize(
         doc, {"algorithm": "URDNA2015", "format": "application/nquads"}
     )

app/lookup.py

@@ -10,6 +10,7 @@ from app.source import _MENTION_REGEX
 
 
 async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
+    query = query.strip()
     if query.startswith("@") or _MENTION_REGEX.match("@" + query):
         query = await webfinger.get_actor_url(query)  # type: ignore  # None check below
 
@@ -37,4 +38,9 @@ async def lookup(db_session: AsyncSession, query: str) -> Actor | RemoteObject:
     if ap.as_list(ap_obj["type"])[0] in ap.ACTOR_TYPES:
         return RemoteActor(ap_obj)
     else:
+        # Some software return objects wrapped in a Create activity (like
+        # python-federation)
+        if ap.as_list(ap_obj["type"])[0] == "Create":
+            ap_obj = await ap.get_object(ap_obj)
+
         return await RemoteObject.from_raw_object(ap_obj)

app/media.py

@@ -1,15 +1,44 @@
 import base64
+import time
 
 from app.config import BASE_URL
+from app.config import hmac_sha256
 
 SUPPORTED_RESIZE = [50, 740]
+EXPIRY_PERIOD = 86400
+EXPIRY_LENGTH = 7
+
+
+class InvalidProxySignatureError(Exception):
+    pass
+
+
+def proxied_media_sig(expires: int, url: str) -> str:
+    hm = hmac_sha256()
+    hm.update(f"{expires}".encode())
+    hm.update(b"|")
+    hm.update(url.encode())
+    return base64.urlsafe_b64encode(hm.digest()).decode()
+
+
+def verify_proxied_media_sig(expires: int, url: str, sig: str) -> None:
+    now = int(time.time() / EXPIRY_PERIOD)
+    expected = proxied_media_sig(expires, url)
+    if now > expires or sig != expected:
+        raise InvalidProxySignatureError("invalid or expired media")
 
 
 def proxied_media_url(url: str) -> str:
     if url.startswith(BASE_URL):
         return url
+    expires = int(time.time() / EXPIRY_PERIOD) + EXPIRY_LENGTH
+    sig = proxied_media_sig(expires, url)
 
-    return "/proxy/media/" + base64.urlsafe_b64encode(url.encode()).decode()
+    return (
+        BASE_URL
+        + f"/proxy/media/{expires}/{sig}/"
+        + base64.urlsafe_b64encode(url.encode()).decode()
+    )
 
 
 def resized_media_url(url: str, size: int) -> str:

app/micropub.py

@@ -132,7 +132,7 @@ async def post_micropub_endpoint(
             h = form_data["h"]
         entry_type = f"h-{h}"
 
-    logger.info(f"Creating {entry_type}")
+    logger.info(f"Creating {entry_type=} with {access_token_info=}")
 
     if entry_type != "h-entry":
         return JSONResponse(
@@ -150,7 +150,7 @@ async def post_micropub_endpoint(
     else:
         content = form_data["content"]
 
-    public_id = await send_create(
+    public_id, _ = await send_create(
         db_session,
         "Note",
         content,

app/models.py

@@ -1,4 +1,5 @@
 import enum
+from datetime import datetime
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -45,7 +46,7 @@ class Actor(Base, BaseActor):
     created_at = Column(DateTime(timezone=True), nullable=False, default=now)
     updated_at = Column(DateTime(timezone=True), nullable=False, default=now)
 
-    ap_id = Column(String, unique=True, nullable=False, index=True)
+    ap_id: Mapped[str] = Column(String, unique=True, nullable=False, index=True)
     ap_actor: Mapped[ap.RawObject] = Column(JSON, nullable=False)
     ap_type = Column(String, nullable=False)
 
@@ -54,6 +55,10 @@ class Actor(Base, BaseActor):
     is_blocked = Column(Boolean, nullable=False, default=False, server_default="0")
     is_deleted = Column(Boolean, nullable=False, default=False, server_default="0")
 
+    are_announces_hidden_from_stream = Column(
+        Boolean, nullable=False, default=False, server_default="0"
+    )
+
     @property
     def is_from_db(self) -> bool:
         return True
@@ -75,7 +80,7 @@ class InboxObject(Base, BaseObject):
 
     ap_actor_id = Column(String, nullable=False)
     ap_type = Column(String, nullable=False, index=True)
-    ap_id = Column(String, nullable=False, unique=True, index=True)
+    ap_id: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
     ap_context = Column(String, nullable=True)
     ap_published_at = Column(DateTime(timezone=True), nullable=False)
     ap_object: Mapped[ap.RawObject] = Column(JSON, nullable=False)
@@ -126,7 +131,7 @@ class InboxObject(Base, BaseObject):
     is_deleted = Column(Boolean, nullable=False, default=False)
     is_transient = Column(Boolean, nullable=False, default=False, server_default="0")
 
-    replies_count = Column(Integer, nullable=False, default=0)
+    replies_count: Mapped[int] = Column(Integer, nullable=False, default=0)
 
     og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 
@@ -158,9 +163,10 @@ class OutboxObject(Base, BaseObject):
     is_hidden_from_homepage = Column(Boolean, nullable=False, default=False)
 
     public_id = Column(String, nullable=False, index=True)
+    slug = Column(String, nullable=True, index=True)
 
     ap_type = Column(String, nullable=False, index=True)
-    ap_id = Column(String, nullable=False, unique=True, index=True)
+    ap_id: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
     ap_context = Column(String, nullable=True)
     ap_object: Mapped[ap.RawObject] = Column(JSON, nullable=False)
 
@@ -176,7 +182,7 @@ class OutboxObject(Base, BaseObject):
 
     likes_count = Column(Integer, nullable=False, default=0)
     announces_count = Column(Integer, nullable=False, default=0)
-    replies_count = Column(Integer, nullable=False, default=0)
+    replies_count: Mapped[int] = Column(Integer, nullable=False, default=0)
     webmentions_count: Mapped[int] = Column(
         Integer, nullable=False, default=0, server_default="0"
     )
@@ -250,6 +256,8 @@ class OutboxObject(Base, BaseObject):
                         "mediaType": attachment.upload.content_type,
                         "name": attachment.alt or attachment.filename,
                         "url": url,
+                        "width": attachment.upload.width,
+                        "height": attachment.upload.height,
                         "proxiedUrl": url,
                         "resizedUrl": BASE_URL
                         + (
@@ -281,6 +289,13 @@ class OutboxObject(Base, BaseObject):
     def is_from_outbox(self) -> bool:
         return True
 
+    @property
+    def url(self) -> str | None:
+        # XXX: rewrite old URL here for compat
+        if self.ap_type == "Article" and self.slug and self.public_id:
+            return f"{BASE_URL}/articles/{self.public_id[:7]}/{self.slug}"
+        return super().url
+
 
 class Follower(Base):
     __tablename__ = "follower"
@@ -422,7 +437,7 @@ class OutboxObjectAttachment(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
 
     upload_id = Column(Integer, ForeignKey("upload.id"), nullable=False)
-    upload = relationship(Upload, uselist=False)
+    upload: Mapped["Upload"] = relationship(Upload, uselist=False)
 
 
 class IndieAuthAuthorizationRequest(Base):
@@ -445,17 +460,53 @@ class IndieAuthAccessToken(Base):
     __tablename__ = "indieauth_access_token"
 
     id = Column(Integer, primary_key=True, index=True)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+    created_at: Mapped[datetime] = Column(
+        DateTime(timezone=True), nullable=False, default=now
+    )
 
     # Will be null for personal access tokens
     indieauth_authorization_request_id = Column(
         Integer, ForeignKey("indieauth_authorization_request.id"), nullable=True
     )
+    indieauth_authorization_request = relationship(
+        IndieAuthAuthorizationRequest,
+        uselist=False,
+    )
 
-    access_token = Column(String, nullable=False, unique=True, index=True)
-    expires_in = Column(Integer, nullable=False)
+    access_token: Mapped[str] = Column(String, nullable=False, unique=True, index=True)
+    refresh_token = Column(String, nullable=True, unique=True, index=True)
+    expires_in: Mapped[int] = Column(Integer, nullable=False)
     scope = Column(String, nullable=False)
     is_revoked = Column(Boolean, nullable=False, default=False)
+    was_refreshed = Column(Boolean, nullable=False, default=False, server_default="0")
+
+
+class OAuthClient(Base):
+    __tablename__ = "oauth_client"
+
+    id = Column(Integer, primary_key=True, index=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=now)
+
+    # Request
+    client_name = Column(String, nullable=False)
+    redirect_uris: Mapped[list[str]] = Column(JSON, nullable=True)
+
+    # Optional from request
+    client_uri = Column(String, nullable=True)
+    logo_uri = Column(String, nullable=True)
+    scope = Column(String, nullable=True)
+
+    # Response
+    client_id = Column(String, nullable=False, unique=True, index=True)
+    client_secret = Column(String, nullable=False, unique=True)
+
+
+@enum.unique
+class WebmentionType(str, enum.Enum):
+    UNKNOWN = "unknown"
+    LIKE = "like"
+    REPLY = "reply"
+    REPOST = "repost"
 
 
 class Webmention(Base):
@@ -474,6 +525,8 @@ class Webmention(Base):
     outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False)
     outbox_object = relationship(OutboxObject, uselist=False)
 
+    webmention_type = Column(Enum(WebmentionType), nullable=True)
+
     @property
     def as_facepile_item(self) -> webmentions.Webmention | None:
         if not self.source_microformats:
@@ -483,6 +536,7 @@ class Webmention(Base):
                 self.source_microformats["items"], self.source
             )
         except Exception:
+            # TODO: return a facepile with the unknown image
             logger.warning(
                 f"Failed to generate facefile item for Webmention id={self.id}"
             )
@@ -537,6 +591,8 @@ class NotificationType(str, enum.Enum):
     FOLLOW_REQUEST_ACCEPTED = "follow_request_accepted"
     FOLLOW_REQUEST_REJECTED = "follow_request_rejected"
 
+    MOVE = "move"
+
     LIKE = "like"
     UNDO_LIKE = "undo_like"
 
@@ -549,6 +605,14 @@ class NotificationType(str, enum.Enum):
     UPDATED_WEBMENTION = "updated_webmention"
     DELETED_WEBMENTION = "deleted_webmention"
 
+    # incoming
+    BLOCKED = "blocked"
+    UNBLOCKED = "unblocked"
+
+    # outgoing
+    BLOCK = "block"
+    UNBLOCK = "unblock"
+
 
 class Notification(Base):
     __tablename__ = "notifications"

app/outgoing_activities.py

@@ -67,6 +67,7 @@ async def _send_actor_update_if_needed(
     logger.info("Will send an Update for the local actor")
 
     from app.boxes import allocate_outbox_id
+    from app.boxes import compute_all_known_recipients
     from app.boxes import outbox_object_id
     from app.boxes import save_outbox_object
 
@@ -85,24 +86,8 @@ async def _send_actor_update_if_needed(
 
     # Send the update to the followers collection and all the actor we have ever
     # contacted
-    followers = (
-        (
-            await db_session.scalars(
-                select(models.Follower).options(joinedload(models.Follower.actor))
-            )
-        )
-        .unique()
-        .all()
-    )
-    for rcp in {
-        follower.actor.shared_inbox_url or follower.actor.inbox_url
-        for follower in followers
-    } | {
-        row.recipient
-        for row in await db_session.execute(
-            select(func.distinct(models.OutgoingActivity.recipient).label("recipient"))
-        )
-    }:  # type: ignore
+    recipients = await compute_all_known_recipients(db_session)
+    for rcp in recipients:
         await new_outgoing_activity(
             db_session,
             recipient=rcp,
@@ -166,7 +151,7 @@ def _set_next_try(
     if not outgoing_activity.tries:
         raise ValueError("Should never happen")
 
-    if outgoing_activity.tries == _MAX_RETRIES:
+    if outgoing_activity.tries >= _MAX_RETRIES:
         outgoing_activity.is_errored = True
         outgoing_activity.next_try = None
     else:

app/prune.py

@@ -102,6 +102,8 @@ async def _prune_old_inbox_objects(
                     models.InboxObject.ap_type.in_(["Note"]),
                 )
             ),
+            # Keep Move object as they are linked to notifications
+            models.InboxObject.ap_type.not_in(["Move"]),
             # Filter by retention days
             models.InboxObject.ap_published_at
             < now() - timedelta(days=INBOX_RETENTION_DAYS),

app/redirect.py

@@ -0,0 +1,28 @@
+from fastapi import Request
+
+from app import templates
+from app.database import AsyncSession
+
+
+async def redirect(
+    request: Request,
+    db_session: AsyncSession,
+    url: str,
+) -> templates.TemplateResponse:
+    """
+    Similar to RedirectResponse, but uses a 200 response with HTML.
+
+    Needed for remote redirects on form submission endpoints,
+    since our CSP policy disallows remote form submission.
+    https://github.com/w3c/webappsec-csp/issues/8#issuecomment-810108984
+    """
+    return await templates.render_template(
+        db_session,
+        request,
+        "redirect.html",
+        {
+            "request": request,
+            "url": url,
+        },
+        headers={"Refresh": "0;url=" + url},
+    )

app/scss/main.scss

@@ -13,21 +13,58 @@ $code-highlight-background: #f0f0f0;
 // Load custom theme
 @import "theme.scss";
 
+.primary-color {
+  color: $primary-color;
+}
+
+#admin {
+  .admin-menu {
+    margin-bottom: 30px;
+    padding: 0 20px;
+  }
+}
+
+.empty-state {
+  padding: 20px;
+}
+
+.public-top-menu {
+  margin: 30px 0 0 0;
+}
+
+.width-95 {
+  width: 95%;
+}
+
+.bold {
+  font-weight: bold;
+}
+
+.admin-new {
+  textarea {
+    font-size: 1.2em;
+    width: 95%;
+  }
+}
+
 .show-more-wrapper {
   .p-summary {
     display: inline-block;
   }
-  label {
+  .show-more-btn {
     margin-left: 5px;
   }
-  .show-more-state {
-    display: none;
+  summary {
+    display: inline-block;
   }
-  .show-more-state ~ .obj-content {
-    margin-top: 0;
+  summary::-webkit-details-marker {
+    display: none
   }
-  .show-more-state:checked ~ .obj-content {
-    display: none;
+  &:not([open]) .show-more-btn::after {
+    content: 'show more';
+  }
+  &[open] .show-more-btn::after {
+    content: 'show less';
   }
 }
 .sensitive-attachment {
@@ -61,13 +98,6 @@ blockquote {
     color: $muted-color;
 }
 
-.poll-bar {
-   width:100%;height:20px;
-   line {
-        stroke: $secondary-color;
-   }
-}
-
 .light-background {
     background: $light-background;
 }
@@ -116,6 +146,9 @@ dl {
     strong {
         color: $primary-color;
     }
+    span {
+      color: $muted-color;
+    }
 }
 
 div.highlight {
@@ -182,6 +215,7 @@ a {
   }
 }
 #main {
+    display: flex;
     flex: 1; 
 }
 main {
@@ -189,11 +223,36 @@ main {
     max-width: 1000px;
     margin: 30px auto;
 }
+
+.main-flex {
+    display: flex;
+    flex: 1;
+}
+
+.centered {
+  display: flex;
+  flex: 1;
+  justify-content: center;
+  align-items: center;
+  div {
+    display: block;
+  }
+}
+
 footer {
   width: 100%;
   max-width: 1000px;
   margin: 20px auto;
   color: $muted-color;
+  p {
+    margin: 0;    
+  }
+}
+.tiny-actor-icon {
+  max-width: 24px;
+  max-height: 24px;
+  position: relative;
+  top: 5px;
 }
 .actor-box {
   display: flex;
@@ -217,6 +276,9 @@ footer {
     padding: 0 20px;
     li {
         display: block;
+        span {
+          padding-right:10px;
+        }
     }
 }
 
@@ -251,6 +313,57 @@ footer {
     margin: 20px 0;
 }
 
+.show-hide-sensitive-btn {
+  display:inline-block;
+}
+
+.no-margin-top {
+  margin-top: 0;
+}
+
+.float-right {
+  float: right;
+}
+
+ul.poll-items {
+  list-style-type: none;
+  padding: 0;
+  li {
+    display: block;
+    p {
+      margin: 20px 0 10px 0;
+      .poll-vote {
+        padding-left: 20px;
+      }
+    }
+
+    .poll-bar {
+       width:100%;height:20px;
+       line {
+            stroke: $secondary-color;
+            stroke-width: 20px;
+       }
+    }
+
+  }
+}
+
+.attachment-wrapper {
+  .attachment-item {
+    margin-top: 20px;
+  }
+  img.attachment {
+    margin: 0;
+  }
+  a.attachment {
+    display: inline-block;
+    margin-bottom: 15px;
+  }
+  audio.attachment {
+    width: 480px;
+  }
+}
+
 nav {
     form {
         margin: 15px 0;
@@ -278,7 +391,7 @@ nav.flexbox {
             margin-right: 0px;
         }
     }
-    a {
+    a:not(.label-btn) {
         color: $primary-color;
         text-decoration: none;
         &:hover, &:active {
@@ -286,35 +399,40 @@ nav.flexbox {
             text-decoration: underline;
         }
     }
-    a.active {
+    a.active:not(.label-btn) {
         color: $secondary-color;
         font-weight: bold;
     }
 }
 
+// after nav.flexbox to override default behavior
+a.label-btn {
+  color: $form-text-color;
+  &:hover {
+    text-decoration: none;
+    color: $form-text-color;
+  }
+}
+
 .ap-object {
     margin: 15px 0;
     padding: 20px;
-    .in-reply-to {
-        color: $muted-color;
-        &:hover {
-            color: $secondary-color;
-            text-decoration: underline;
-        }
-    }
     nav {
         color: $muted-color;
     }
+    .in-reply-to {
+        display: inline;
+        color: $muted-color;
+    }
     .e-content, .activity-og-meta {
         a:hover {
             text-decoration: underline;
         }
     }
     .activity-attachment {
-        margin: 30px 0;
+        margin: 30px 0 20px 0;
         img, audio, video {
-            width: 100%;
-            max-width: 740px;
+            max-width: calc(min(740px, 100%));
         }
     }
     img.inline-img {
@@ -322,11 +440,25 @@ nav.flexbox {
         max-width: 740px;
     }
 }
+
+.activity-og-meta {
+  display: flex;
+  column-gap: 20px;
+  margin: 20px 0;
+  img {
+    max-width: 200px;
+    max-height: 100px;
+  }
+  small {
+    display: block;
+  }
+}
+
 .ap-object-expanded {
     border: 2px dashed $secondary-color;
 }
 
-.error-box {
+.error-box, .scolor {
     color: $secondary-color;
 }
 
@@ -337,6 +469,9 @@ nav.flexbox {
   span {
     color: $muted-color;
   }
+  span.new {
+    color: $secondary-color;
+  }
 }
 .actor-metadata {
     color: $muted-color;
@@ -344,3 +479,93 @@ nav.flexbox {
 .emoji, .custom-emoji {
   max-width: 25px;
 }
+
+.indieauth-box {
+  display: flex;
+  column-gap: 20px;
+
+  .indieauth-logo {
+    flex: initial;
+    width: 100px;
+    img {
+      max-width: 100px;
+    }
+  }
+  .indieauth-details {
+    flex: 1;
+    div {
+      padding-left: 20px;
+      a {
+        font-size: 1.2em;
+        font-weight: 600;
+      }
+    }
+  }
+}
+
+.public-interactions {
+  display: flex;
+  column-gap: 20px;
+  flex-wrap: wrap;
+  margin-top: 20px;
+  .interactions-block {
+    flex: 0 1 30%;
+    max-width: 50%;
+    .facepile-wrapper {
+      display: flex;
+      column-gap: 20px;
+      row-gap: 20px;
+      flex-wrap: wrap;
+      margin-top: 20px;
+      a {
+        height: 50px;
+        img {
+          max-width: 50px;
+        }
+      }
+      .and-x-more {
+        display: inline-block;
+        align-self: center;
+      }
+    }
+  }
+}
+
+.error-title {
+  a {
+    text-decoration: underline;
+  }
+}
+
+.ap-place {
+  h3 {
+    display: inline;
+    font-weight: normal;
+  }
+  h3::after {
+    content: ': ';
+  }
+}
+
+.margin-top-20 {
+  margin-top: 20px;
+}
+
+.video-wrapper {
+    position: relative;
+}
+
+.video-gif-overlay {
+    display: none;
+}
+
+.video-gif-mode + .video-gif-overlay {
+    display: block;
+    position: absolute;
+    top: 5px;
+    left: 5px;
+    padding: 0 3px;
+    font-size: 0.8em;
+    background: rgba(0,0,0,.5);
+    color: #fff;
+}

app/source.py

@@ -1,54 +1,146 @@
 import re
+import typing
 
-from markdown import markdown
+from loguru import logger
+from mistletoe import Document  # type: ignore
+from mistletoe.block_token import CodeFence  # type: ignore
+from mistletoe.html_renderer import HTMLRenderer  # type: ignore
+from mistletoe.span_token import SpanToken  # type: ignore
+from pygments.formatters import HtmlFormatter  # type: ignore
+from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
+from pygments.util import ClassNotFound  # type: ignore
 from sqlalchemy import select
 
-from app import models
 from app import webfinger
-from app.actor import Actor
-from app.actor import fetch_actor
 from app.config import BASE_URL
+from app.config import CODE_HIGHLIGHTING_THEME
 from app.database import AsyncSession
 from app.utils import emoji
 
+if typing.TYPE_CHECKING:
+    from app.actor import Actor
 
-def _set_a_attrs(attrs, new=False):
-    attrs[(None, "target")] = "_blank"
-    attrs[(None, "class")] = "external"
-    attrs[(None, "rel")] = "noopener"
-    attrs[(None, "title")] = attrs[(None, "href")]
-    return attrs
-
-
+_FORMATTER = HtmlFormatter(style=CODE_HIGHLIGHTING_THEME)
 _HASHTAG_REGEX = re.compile(r"(#[\d\w]+)")
-_MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+")
+_MENTION_REGEX = re.compile(r"(@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+)")
+_URL_REGEX = re.compile(
+    "(https?:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*))"  # noqa: E501
+)
 
 
-async def _hashtagify(
-    db_session: AsyncSession, content: str
-) -> tuple[str, list[dict[str, str]]]:
-    tags = []
-    hashtags = re.findall(_HASHTAG_REGEX, content)
-    hashtags = sorted(set(hashtags), reverse=True)  # unique tags, longest first
-    for hashtag in hashtags:
-        tag = hashtag[1:]
-        link = f'<a href="{BASE_URL}/t/{tag}" class="mention hashtag" rel="tag">#<span>{tag}</span></a>'  # noqa: E501
-        tags.append(dict(href=f"{BASE_URL}/t/{tag}", name=hashtag, type="Hashtag"))
-        content = content.replace(hashtag, link)
-    return content, tags
+class AutoLink(SpanToken):
+    parse_inner = False
+    precedence = 1
+    pattern = _URL_REGEX
+
+    def __init__(self, match_obj: re.Match) -> None:
+        self.target = match_obj.group()
 
 
-async def _mentionify(
+class Mention(SpanToken):
+    parse_inner = False
+    precedence = 10
+    pattern = _MENTION_REGEX
+
+    def __init__(self, match_obj: re.Match) -> None:
+        self.target = match_obj.group()
+
+
+class Hashtag(SpanToken):
+    parse_inner = False
+    precedence = 10
+    pattern = _HASHTAG_REGEX
+
+    def __init__(self, match_obj: re.Match) -> None:
+        self.target = match_obj.group()
+
+
+class CustomRenderer(HTMLRenderer):
+    def __init__(
+        self,
+        mentioned_actors: dict[str, "Actor"] = {},
+        enable_mentionify: bool = True,
+        enable_hashtagify: bool = True,
+    ) -> None:
+        extra_tokens = []
+        if enable_mentionify:
+            extra_tokens.append(Mention)
+        if enable_hashtagify:
+            extra_tokens.append(Hashtag)
+        super().__init__(AutoLink, *extra_tokens)
+
+        self.tags: list[dict[str, str]] = []
+        self.mentioned_actors = mentioned_actors
+
+    def render_auto_link(self, token: AutoLink) -> str:
+        template = '<a href="{target}" rel="noopener">{inner}</a>'
+        target = self.escape_url(token.target)
+        return template.format(target=target, inner=target)
+
+    def render_mention(self, token: Mention) -> str:
+        mention = token.target
+        suffix = ""
+        if mention.endswith("."):
+            mention = mention[:-1]
+            suffix = "."
+        actor = self.mentioned_actors.get(mention)
+        if not actor:
+            return mention
+
+        self.tags.append(dict(type="Mention", href=actor.ap_id, name=mention))
+
+        link = f'<span class="h-card"><a href="{actor.url}" class="u-url mention">{actor.handle}</a></span>{suffix}'  # noqa: E501
+        return link
+
+    def render_hashtag(self, token: Hashtag) -> str:
+        tag = token.target[1:]
+        link = f'<a href="{BASE_URL}/t/{tag.lower()}" class="mention hashtag" rel="tag">#<span>{tag}</span></a>'  # noqa: E501
+        self.tags.append(
+            dict(
+                href=f"{BASE_URL}/t/{tag.lower()}",
+                name=token.target.lower(),
+                type="Hashtag",
+            )
+        )
+        return link
+
+    def render_block_code(self, token: CodeFence) -> str:
+        lexer_attr = ""
+        try:
+            lexer = get_lexer(token.language)
+            lexer_attr = f' data-microblogpub-lexer="{lexer.aliases[0]}"'
+        except ClassNotFound:
+            pass
+
+        code = token.children[0].content
+        return f"<pre><code{lexer_attr}>\n{code}\n</code></pre>"
+
+
+async def _prefetch_mentioned_actors(
     db_session: AsyncSession,
     content: str,
-) -> tuple[str, list[dict[str, str]], list[Actor]]:
-    tags = []
-    mentioned_actors = []
+) -> dict[str, "Actor"]:
+    from app import models
+    from app.actor import fetch_actor
+
+    actors = {}
+
     for mention in re.findall(_MENTION_REGEX, content):
+        if mention in actors:
+            continue
+
+        # XXX: the regex catches stuff like `@toto@example.com.`
+        if mention.endswith("."):
+            mention = mention[:-1]
+
+        try:
             _, username, domain = mention.split("@")
             actor = (
                 await db_session.execute(
-                select(models.Actor).where(models.Actor.handle == mention)
+                    select(models.Actor).where(
+                        models.Actor.handle == mention,
+                        models.Actor.is_deleted.is_(False),
+                    )
                 )
             ).scalar_one_or_none()
             if not actor:
@@ -58,36 +150,69 @@ async def _mentionify(
                     continue
                 actor = await fetch_actor(db_session, actor_url)
 
-        mentioned_actors.append(actor)
-        tags.append(dict(type="Mention", href=actor.ap_id, name=mention))
+            actors[mention] = actor
+        except Exception:
+            logger.exception(f"Failed to prefetch {mention}")
 
-        link = f'<span class="h-card"><a href="{actor.url}" class="u-url mention">{actor.handle}</a></span>'  # noqa: E501
-        content = content.replace(mention, link)
-    return content, tags, mentioned_actors
+    return actors
+
+
+def hashtagify(
+    content: str,
+) -> tuple[str, list[dict[str, str]]]:
+    tags = []
+    with CustomRenderer(
+        mentioned_actors={},
+        enable_mentionify=False,
+        enable_hashtagify=True,
+    ) as renderer:
+        rendered_content = renderer.render(Document(content))
+        tags.extend(renderer.tags)
+
+    # Handle custom emoji
+    tags.extend(emoji.tags(content))
+
+    return rendered_content, tags
 
 
 async def markdownify(
     db_session: AsyncSession,
     content: str,
-    mentionify: bool = True,
-    hashtagify: bool = True,
-) -> tuple[str, list[dict[str, str]], list[Actor]]:
+    enable_mentionify: bool = True,
+    enable_hashtagify: bool = True,
+) -> tuple[str, list[dict[str, str]], list["Actor"]]:
     """
     >>> content, tags = markdownify("Hello")
 
     """
     tags = []
-    mentioned_actors: list[Actor] = []
-    if hashtagify:
-        content, hashtag_tags = await _hashtagify(db_session, content)
-        tags.extend(hashtag_tags)
-    if mentionify:
-        content, mention_tags, mentioned_actors = await _mentionify(db_session, content)
-        tags.extend(mention_tags)
+    mentioned_actors: dict[str, "Actor"] = {}
+    if enable_mentionify:
+        mentioned_actors = await _prefetch_mentioned_actors(db_session, content)
+
+    with CustomRenderer(
+        mentioned_actors=mentioned_actors,
+        enable_mentionify=enable_mentionify,
+        enable_hashtagify=enable_hashtagify,
+    ) as renderer:
+        rendered_content = renderer.render(Document(content))
+        tags.extend(renderer.tags)
 
     # Handle custom emoji
     tags.extend(emoji.tags(content))
 
-    content = markdown(content, extensions=["mdx_linkify", "fenced_code"])
+    return rendered_content, dedup_tags(tags), list(mentioned_actors.values())
 
-    return content, tags, mentioned_actors
+
+def dedup_tags(tags: list[dict[str, str]]) -> list[dict[str, str]]:
+    idx = set()
+    deduped_tags = []
+    for tag in tags:
+        tag_idx = (tag["type"], tag["name"])
+        if tag_idx in idx:
+            continue
+
+        idx.add(tag_idx)
+        deduped_tags.append(tag)
+
+    return deduped_tags

app/static/common-admin.js

@@ -0,0 +1,11 @@
+document.addEventListener('DOMContentLoaded', (ev) => {
+  // Add confirm to "delete" button next to outbox objects
+  var forms = document.getElementsByClassName("object-delete-form")
+  for (var i = 0; i < forms.length; i++) {
+    forms[i].addEventListener('submit', (ev) => {
+      if (!confirm('Do you really want to delete this object?')) {
+        ev.preventDefault();
+      };
+    });
+  }
+});

app/static/common.js

@@ -0,0 +1,32 @@
+function hasAudio (video) {
+    return video.mozHasAudio ||
+    Boolean(video.webkitAudioDecodedByteCount) ||
+    Boolean(video.audioTracks && video.audioTracks.length);
+}
+
+function setVideoInGIFMode(video) {
+    if (!hasAudio(video)) {
+        if (typeof video.loop == 'boolean' && video.duration <= 10.0) {
+            video.classList.add("video-gif-mode");
+            video.loop = true;
+            video.controls = false;
+            video.addEventListener("mouseover", () => {
+                video.play();
+            })
+            video.addEventListener("mouseleave", () => {
+                video.pause();
+            })
+        }
+    };
+}
+
+var items = document.getElementsByTagName("video")
+for (var i = 0; i < items.length; i++) {
+    if (items[i].duration) {
+        setVideoInGIFMode(items[i]);
+    } else {
+        items[i].addEventListener("loadeddata", function() {
+            setVideoInGIFMode(this);
+        });
+    }
+}

app/templates.py

@@ -1,4 +1,3 @@
-import base64
 from datetime import datetime
 from datetime import timezone
 from functools import lru_cache
@@ -26,8 +25,9 @@ from app.actor import LOCAL_ACTOR
 from app.ap_object import Attachment
 from app.ap_object import Object
 from app.config import BASE_URL
-from app.config import CSS_HASH
+from app.config import CUSTOM_FOOTER
 from app.config import DEBUG
+from app.config import SESSION_TIMEOUT
 from app.config import VERSION
 from app.config import generate_csrf_token
 from app.config import session_serializer
@@ -39,7 +39,7 @@ from app.utils.highlight import HIGHLIGHT_CSS
 from app.utils.highlight import highlight
 
 _templates = Jinja2Templates(
-    directory="app/templates",
+    directory=["data/templates", "app/templates"],  # type: ignore  # bad typing
     trim_blocks=True,
     lstrip_blocks=True,
 )
@@ -59,13 +59,8 @@ def _filter_domain(text: str) -> str:
 
 def _media_proxy_url(url: str | None) -> str:
     if not url:
-        return "/static/nopic.png"
-
-    if url.startswith(BASE_URL):
-        return url
-
-    encoded_url = base64.urlsafe_b64encode(url.encode()).decode()
-    return f"/proxy/media/{encoded_url}"
+        return BASE_URL + "/static/nopic.png"
+    return proxied_media_url(url)
 
 
 def is_current_user_admin(request: Request) -> bool:
@@ -75,10 +70,10 @@ def is_current_user_admin(request: Request) -> bool:
         try:
             loaded_session = session_serializer.loads(
                 session_cookie,
-                max_age=3600 * 12,
+                max_age=SESSION_TIMEOUT,
             )
         except Exception:
-            pass
+            logger.exception("Failed to validate session timeout")
         else:
             is_admin = loaded_session.get("is_logged_in")
 
@@ -90,6 +85,8 @@ async def render_template(
     request: Request,
     template: str,
     template_args: dict[str, Any] | None = None,
+    status_code: int = 200,
+    headers: dict[str, str] | None = None,
 ) -> TemplateResponse:
     if template_args is None:
         template_args = {}
@@ -103,7 +100,6 @@ async def render_template(
             "request": request,
             "debug": DEBUG,
             "microblogpub_version": VERSION,
-            "css_hash": CSS_HASH,
             "is_admin": is_admin,
             "csrf_token": generate_csrf_token(),
             "highlight_css": HIGHLIGHT_CSS,
@@ -131,8 +127,11 @@ async def render_template(
                 select(func.count(models.Following.id))
             ),
             "actor_types": ap.ACTOR_TYPES,
+            "custom_footer": CUSTOM_FOOTER,
             **template_args,
         },
+        status_code=status_code,
+        headers=headers,
     )
 
 
@@ -289,6 +288,10 @@ ALLOWED_ATTRIBUTES: dict[str, list[str] | Callable[[str, str, str], bool]] = {
 }
 
 
+def _allow_all_attributes(tag: Any, name: Any, value: Any) -> bool:
+    return True
+
+
 @lru_cache(maxsize=256)
 def _update_inline_imgs(content):
     soup = BeautifulSoup(content, "html5lib")
@@ -318,7 +321,11 @@ def _clean_html(html: str, note: Object) -> str:
                         _update_inline_imgs(highlight(html))
                     ),
                     tags=ALLOWED_TAGS,
-                    attributes=ALLOWED_ATTRIBUTES,
+                    attributes=(
+                        _allow_all_attributes
+                        if note.ap_id.startswith(config.ID)
+                        else ALLOWED_ATTRIBUTES
+                    ),
                     strip=True,
                 ),
                 note,
@@ -329,6 +336,14 @@ def _clean_html(html: str, note: Object) -> str:
         raise
 
 
+def _clean_html_wm(html: str) -> str:
+    return bleach.clean(
+        html,
+        attributes=ALLOWED_ATTRIBUTES,
+        strip=True,
+    )
+
+
 def _timeago(original_dt: datetime) -> str:
     dt = original_dt
     if dt.tzinfo:
@@ -377,8 +392,8 @@ def _html2text(content: str) -> str:
 
 
 def _replace_emoji(u: str, _) -> str:
-    filename = hex(ord(u))[2:]
-    return config.EMOJI_TPL.format(filename=filename, raw=u)
+    filename = "-".join(hex(ord(c))[2:] for c in u)
+    return config.EMOJI_TPL.format(base_url=BASE_URL, filename=filename, raw=u)
 
 
 def _emojify(text: str, is_local: bool) -> str:
@@ -405,6 +420,7 @@ def _poll_item_pct(item: ap.RawObject, voters_count: int) -> int:
 _templates.env.filters["domain"] = _filter_domain
 _templates.env.filters["media_proxy_url"] = _media_proxy_url
 _templates.env.filters["clean_html"] = _clean_html
+_templates.env.filters["clean_html_wm"] = _clean_html_wm
 _templates.env.filters["timeago"] = _timeago
 _templates.env.filters["format_date"] = _format_date
 _templates.env.filters["has_media_type"] = _has_media_type
@@ -414,3 +430,10 @@ _templates.env.filters["pluralize"] = _pluralize
 _templates.env.filters["parse_datetime"] = _parse_datetime
 _templates.env.filters["poll_item_pct"] = _poll_item_pct
 _templates.env.filters["privacy_replace_url"] = privacy_replace.replace_url
+_templates.env.globals["JS_HASH"] = config.JS_HASH
+_templates.env.globals["CSS_HASH"] = config.CSS_HASH
+_templates.env.globals["BASE_URL"] = config.BASE_URL
+_templates.env.globals["HIDES_FOLLOWERS"] = config.HIDES_FOLLOWERS
+_templates.env.globals["HIDES_FOLLOWING"] = config.HIDES_FOLLOWING
+_templates.env.globals["NAVBAR_ITEMS"] = config.NavBarItems
+_templates.env.globals["ICON_URL"] = config.CONFIG.icon_url

app/templates/admin_direct_messages.html

@@ -10,7 +10,9 @@
 {% for anybox_object, convo, actors in threads %}
     <div class="actor-action">
     With {% for actor in actors %}
-        <a href="">{{ actor.handle }}</a>
+    <a href="{{ url_for("admin_profile") }}?actor_id={{ actor.ap_id }}">
+        {{ actor.handle }}
+    </a>
     {% endfor %}
     </div>
     {{ utils.display_object(anybox_object) }}

app/templates/admin_inbox.html

@@ -19,7 +19,7 @@
 
 {% for inbox_object in inbox %}
 {% if inbox_object.ap_type == "Announce" %}
-    {{ utils.actor_action(inbox_object, "shared") }}
+    {{ utils.actor_action(inbox_object, "shared", with_icon=True) }}
     {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
 {% elif inbox_object.ap_type in ["Article", "Note", "Video", "Page", "Question"] %}
 {{ utils.display_object(inbox_object) }}
@@ -27,7 +27,7 @@
     {{ utils.actor_action(inbox_object, "followed you") }}
     {{ utils.display_actor(inbox_object.actor, actors_metadata) }}
 {% elif inbox_object.ap_type == "Like" %}
-    {{ utils.actor_action(inbox_object, "liked one of your post") }}
+    {{ utils.actor_action(inbox_object, "liked one of your posts", with_icon=True) }}
     {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
 {% else %}
     <p>

app/templates/admin_new.html

@@ -25,7 +25,7 @@
 </nav>
 
 
-<form class="form" action="{{ request.url_for("admin_actions_new") }}" enctype="multipart/form-data" method="POST">
+<form class="form admin-new" action="{{ request.url_for("admin_actions_new") }}" enctype="multipart/form-data" method="POST">
     {{ utils.embed_csrf_token() }}
     {{ utils.embed_redirect_url() }}
     <p>
@@ -38,7 +38,7 @@
 
     {% if request.query_params.type == "Article" %}
     <p>
-        <input type="text" style="width:95%" name="name" placeholder="Title">     
+        <input type="text" class="width-95" name="name" placeholder="Title">
     </p>
     {% endif %}
 
@@ -49,7 +49,7 @@
     <span class="ji"><img src="{{ emoji.icon.url }}" alt="{{ emoji.name }}" title="{{ emoji.name }}" class="custom-emoji"></span>
     {% endfor %}
 
-    <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" placeholder="Hey!" style="font-size:1.2em;width:95%;">{{ content }}</textarea>
+    <textarea name="content" rows="10" cols="50" autofocus="autofocus" designMode="on" placeholder="Hey!">{{ content }}</textarea>
 
     {% if request.query_params.type == "Question" %}
         <p>
@@ -69,20 +69,20 @@
         </p>
         {% for i in ["1", "2", "3", "4"] %}
             <p>
-            <input type="text" name="poll_answer_{{ i }}" style="width:95%;" placeholder="Option {{ i }}, leave empty to disable">
+            <input type="text" name="poll_answer_{{ i }}" class="width-95" placeholder="Option {{ i }}, leave empty to disable">
             </p>
         {% endfor %}
     {% endif %}
 
     <p>
-    <input type="text" name="content_warning" placeholder="content warning (will mark the post as sensitive)"{% if content_warning %} value="{{ content_warning }}"{% endif %} style="width:95%;">
+    <input type="text" name="content_warning" placeholder="content warning (will mark the post as sensitive)"{% if content_warning %} value="{{ content_warning }}"{% endif %} class="width-95">
     </p>
     <p>
         <input type="checkbox" name="is_sensitive" id="is_sensitive"> <label for="is_sensitive">Mark attachment(s) as sensitive</label>
     </p>
     <input type="hidden" name="in_reply_to" value="{{ request.query_params.in_reply_to }}">
     <p>
-        <input id="files" name="files" type="file" multiple style="width:95%;">
+        <input id="files" name="files" type="file" class="width-95" multiple>
     </p>
     <div id="alts"></div>
     <p>
@@ -90,5 +90,5 @@
     </p>
 </form>
 </div>
-<script src="/static/new.js"></script>
+<script src="{{ BASE_URL }}/static/new.js?v={{ JS_HASH }}"></script>
 {% endblock %}

app/templates/admin_outbox.html

@@ -12,18 +12,16 @@
 {% for outbox_object in outbox %}
 
     {% if outbox_object.ap_type == "Announce" %}
-        <div class="actor-action">You shared</div>
+        <div class="actor-action">You shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
     {% elif outbox_object.ap_type == "Like" %}
-        <div class="actor-action">You liked</div>
+        <div class="actor-action">You liked <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_object(outbox_object.relates_to_anybox_object) }}
     {% elif outbox_object.ap_type == "Follow" %}
-        <div class="actor-action">You followed</div>
+        <div class="actor-action">You followed <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
         {{ utils.display_actor(outbox_object.relates_to_actor, actors_metadata) }}
     {% elif outbox_object.ap_type in ["Article", "Note", "Video", "Question"] %}
         {{ utils.display_object(outbox_object) }}
-    {% else %}
-        Implement {{ outbox_object.ap_type }}
     {% endif %}
 
 {% endfor %}

app/templates/admin_profile.html

@@ -9,10 +9,21 @@
     {{ utils.display_actor(actor, actors_metadata, with_details=True) }}
     {% for inbox_object in inbox_objects %}
         {% if inbox_object.ap_type == "Announce" %}
-            {{ utils.actor_action(inbox_object, "shared") }}
+            {{ utils.actor_action(inbox_object, "shared", with_icon=True) }}
             {{ utils.display_object(inbox_object.relates_to_anybox_object) }}
         {% else %}
                 {{ utils.display_object(inbox_object) }}
         {% endif %}
     {% endfor %}
+
+{% if next_cursor %}
+<div class="box">
+    <p>
+    <a href="{{ request.url._path }}?actor_id={{ request.query_params.actor_id }}&cursor={{ next_cursor }}">
+            See more
+        </a>
+    </p>
+</div>
+{% endif %}
+
 {% endblock %}

app/templates/articles.html

@@ -11,8 +11,8 @@
 <ul class="h-feed" id="articles">
 <data class="p-name" value="{{ local_actor.display_name}}'s articles"></data>
 {% for outbox_object in objects %}
-    <li>
-        <span class="muted" style="padding-right:10px;">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</span> <a href="{{ outbox_object.url }}">{{ outbox_object.name }}</a>
+    <li class="h-entry">
+        <time class="muted dt-published" datetime="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at.strftime("%b %d, %Y") }}</time> <a href="{{ outbox_object.url }}" class="u-url u-uid p-name">{{ outbox_object.name }}</a>
     </li>
 {% endfor %}
 </ul>

app/templates/custom_page.html

@@ -0,0 +1,30 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ title }}</title>
+{% if request.url.path == "/" %}
+<link rel="indieauth-metadata" href="{{ url_for("well_known_authorization_server") }}">
+<link rel="authorization_endpoint" href="{{ url_for("indieauth_authorization_endpoint") }}">
+<link rel="token_endpoint" href="{{ url_for("indieauth_token_endpoint") }}">
+<link rel="micropub" href="{{ url_for("micropub_endpoint") }}">
+<link rel="alternate" href="{{ local_actor.url }}" title="ActivityPub profile"  type="application/activity+json">
+<meta content="profile" property="og:type" />
+<meta content="{{ local_actor.url }}" property="og:url" />
+<meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
+<meta content="Homepage" property="og:title" />
+<meta content="{{ local_actor.summary | html2text | trim }}" property="og:description" />
+<meta content="{{ ICON_URL }}" property="og:image" />
+<meta content="summary" property="twitter:card" />
+<meta content="{{ local_actor.handle }}" property="profile:username" />
+{% endif %}
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    {{ page_content | safe }}
+</div>
+
+{% endblock %}

app/templates/error.html

@@ -0,0 +1,12 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+{% block main_tag %} class="main-flex"{% endblock %}
+{% block head %}
+<title>{{ title }}</title>
+{% endblock %}
+
+{% block content %}
+<div class="centered primary-color box">
+    <h1 class="error-title">{{ title | safe }}</h1>
+</div>
+{% endblock %}

app/templates/followers.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s followers</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/following.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>{{ local_actor.display_name }}'s follows</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/header.html

@@ -25,20 +25,35 @@
 </div>
 
 {%- macro header_link(url, text) -%}
-{% set url_for = request.app.router.url_path_for(url) %}
-<a href="{{ url_for }}" {% if request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
+{% set url_for = BASE_URL + request.app.router.url_path_for(url) %}
+<a href="{{ url_for }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
 {% endmacro %}
 
-<div style="margin:30px 0 0 0;">
+{%- macro navbar_item_link(navbar_item) -%}
+{% set url_for = BASE_URL + navbar_item[0] %}
+<a href="{{ navbar_item[0] }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ navbar_item[1] }}</a>
+{% endmacro %}
+
+<div class="public-top-menu">
 <nav class="flexbox">
     <ul>
+        {% if NAVBAR_ITEMS.INDEX_NAVBAR_ITEM %}
+        <li>{{ navbar_item_link(NAVBAR_ITEMS.INDEX_NAVBAR_ITEM) }}</li>
+        {% endif %}
         <li>{{ header_link("index", "Notes") }}</li>
         {% if articles_count %}
             <li>{{ header_link("articles", "Articles") }}</li>
         {% endif %}
+        {% if not HIDES_FOLLOWERS or is_admin %}
         <li>{{ header_link("followers", "Followers") }} <span class="counter">{{ followers_count }}</span></li>
+        {% endif %}
+        {% if not HIDES_FOLLOWING or is_admin %}
         <li>{{ header_link("following", "Following") }} <span class="counter">{{ following_count }}</span></li>
+        {% endif %}
         <li>{{ header_link("get_remote_follow", "Remote follow") }}</li>
+        {% for navbar_item in NAVBAR_ITEMS.EXTRA_NAVBAR_ITEMS %}
+            {{ navbar_item_link(navbar_item) }}
+        {% endfor %}
     </ul>
 </nav>
 </div>

app/templates/index.html

@@ -13,7 +13,7 @@
 <meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
 <meta content="Homepage" property="og:title" />
 <meta content="{{ local_actor.summary | html2text | trim }}" property="og:description" />
-<meta content="{{ local_actor.url }}" property="og:image" />
+<meta content="{{ ICON_URL }}" property="og:image" />
 <meta content="summary" property="twitter:card" />
 <meta content="{{ local_actor.handle }}" property="profile:username" />
 {% endblock %}
@@ -21,19 +21,26 @@
 {% block content %}
 {% include "header.html" %}
 
-<div class="h-feed">
-<data class="p-name" value="{{ local_actor.display_name}}'s notes"></data>
-{% for outbox_object in objects %}
-{% if outbox_object.ap_type in ["Note", "Article", "Video", "Question"] %}
-{{ utils.display_object(outbox_object) }}
-{% elif outbox_object.ap_type == "Announce" %}
-<div class="shared-header"><strong>{{ local_actor.display_name }}</strong> shared</div>
-{{ utils.display_object(outbox_object.relates_to_anybox_object) }}
-{% endif %}
-{% endfor %}
-</div>
+{% if objects %}
 
-<div class="box">
+    <div class="h-feed">
+    <data class="p-name" value="{{ local_actor.display_name}}'s notes"></data>
+    {% for outbox_object in objects %}
+    {% if outbox_object.ap_type in ["Note", "Video", "Question"] %}
+    {{ utils.display_object(outbox_object) }}
+    {% elif outbox_object.ap_type == "Announce" %}
+    <div class="h-entry" id="{{ outbox_object.permalink_id }}">
+        <div class="shared-header"><strong><a class="p-author h-card" href="{{ local_actor.url }}">{{ utils.display_tiny_actor_icon(local_actor) }}  {{ local_actor.display_name | clean_html(local_actor) | safe }}</a></strong> shared <span title="{{ outbox_object.ap_published_at.isoformat() }}">{{ outbox_object.ap_published_at | timeago }}</span></div>
+        <div class="h-cite u-repost-of">
+            {{ utils.display_object(outbox_object.relates_to_anybox_object, is_h_entry=False) }}
+        </div>
+    </div>
+    {% endif %}
+    {% endfor %}
+    </div>
+
+    {% if has_previous_page or has_next_page %}
+        <div class="box">
             {% if has_previous_page %}
             <a href="{{ url_for("index") }}?page={{ current_page - 1 }}">Previous</a>
             {% endif %}
@@ -41,6 +48,13 @@
             {% if has_next_page %}
             <a href="{{ url_for("index") }}?page={{ current_page + 1 }}">Next</a>
             {% endif %}
-</div>
+        </div>
+    {% endif %}
+
+{% else %}
+    <div class="empty-state">
+        <p>Nothing to see here yet!</p>
+    </div>
+{% endif %}
 
 {% endblock %}

app/templates/indieauth_flow.html

@@ -2,16 +2,20 @@
 {% extends "layout.html" %}
 {% block content %}
 <div class="box">
-    <div style="display:flex;column-gap: 20px;">
+    <div class"indieauth-box">
         {% if client.logo %}
-        <div style="flex:initial;width:100px;">
-            <img src="{{client.logo | media_proxy_url }}" style="max-width:100px;" alt="{{ client.name }} logo">
+        <div class="indieauth-logo">
+            <img src="{{client.logo | media_proxy_url }}" alt="{{ client.name }} logo">
         </div>
         {% endif %}
-        <div style="flex:1;">
-            <div style="padding-left: 20px;">
-                <a class="lcolor" style="font-size:1.2em;font-weight:600;" href="{{ client.url }}">{{ client.name }}</a>
-                <p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
+        <div class="indieauth-details">
+            <div>
+                {% if client.url %}
+                    <a class="scolor" href="{{ client.url }}">{{ client.name }}</a>
+                {% else %}
+                    <span class="scolor">{{ client.name }}</span>
+                {% endif %}
+                <p>wants you to login{% if me %} as <strong class="lcolor">{{ me }}</strong>{% endif %} with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
 
 
                 <form method="POST" action="{{ url_for('indieauth_flow') }}" class="form">

app/templates/layout.html

@@ -4,26 +4,24 @@
 <meta charset="utf-8">
 <meta http-equiv="x-ua-compatible" content="ie=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-<link rel="stylesheet" href="/static/css/main.css?v={{ css_hash }}">
+<link rel="stylesheet" href="{{ BASE_URL }}/static/css/main.css?v={{ CSS_HASH }}">
 <link rel="alternate" title="{{ local_actor.display_name}}'s microblog" type="application/json" href="{{ url_for("json_feed") }}" />
 <link rel="alternate" href="{{ url_for("rss_feed") }}" type="application/rss+xml" title="{{ local_actor.display_name}}'s microblog">
 <link rel="alternate" href="{{ url_for("atom_feed") }}" type="application/atom+xml" title="{{ local_actor.display_name}}'s microblog">
-<link rel="icon" type="image/x-icon" href="/static/favicon.ico">
-<style>
-{{ highlight_css }}
-</style>
+<link rel="icon" type="image/x-icon" href="{{ BASE_URL }}/static/favicon.ico">
+<style>{{ highlight_css }}</style>
 {% block head %}{% endblock %}
 </head>
 <body>
 <div id="main">
-<main>
+<main{%- block main_tag %}{%- endblock %}>
 {% if is_admin %}
 <div id="admin">
 {% macro admin_link(url, text) %}
-{% set url_for = request.app.router.url_path_for(url) %}
-<a href="{{ url_for }}" {% if request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
+{% set url_for = BASE_URL + request.app.router.url_path_for(url) %}
+<a href="{{ url_for }}" {% if BASE_URL + request.url.path == url_for %}class="active"{% endif %}>{{ text }}</a>
 {% endmacro %}
-<div style="margin-bottom:30px;padding: 0 20px;">
+<div class="admin-menu">
 <nav class="flexbox">
     <ul>
         <li>{{ admin_link("index", "Public") }}</li>
@@ -47,8 +45,16 @@
 
 <footer class="footer">
     <div class="box">
+    {% if custom_footer %}
+        {{ custom_footer | safe }}
+    {% else %}
         Powered by <a href="https://docs.microblog.pub">microblog.pub</a> <small class="microblogpub-version"><code>{{ microblogpub_version }}</code></small> and the <a href="https://activitypub.rocks/">ActivityPub</a> protocol. <a href="{{ url_for("login") }}">Admin</a>.
+    {% endif %}
     </div>
 </footer>
+{% if is_admin %}
+<script src="{{ BASE_URL }}/static/common-admin.js?v={{ JS_HASH }}"></script>
+{% endif %}
+<script src="{{ BASE_URL }}/static/common.js?v={{ JS_HASH }}"></script>
 </body>
 </html>

app/templates/login.html

@@ -1,14 +1,21 @@
 {%- import "utils.html" as utils with context -%}
 {% extends "layout.html" %}
+{% block head %}
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
+{% block main_tag %} class="main-flex"{% endblock %}
 {% block content %}
-<div style="display:grid;height:80%;">
-<div style="margin:auto;">
-<form class="form" action="/admin/login" method="POST">
-<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-<input type="hidden" name="redirect" value="{{ redirect }}">
-<input type="password" placeholder="password" name="password" autofocus>
-<input type="submit" value="login">
-</form>
-</div>
+<div class="centered">
+    <div>
+        {% if error %}
+            <p class="primary-color">Invalid password.</p>
+        {% endif %}
+        <form class="form" action="{{ BASE_URL }}/admin/login" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+            <input type="hidden" name="redirect" value="{{ redirect }}">
+            <input type="password" placeholder="password" name="password" autofocus>
+            <input type="submit" value="login">
+        </form>
+    </div>
 </div>
 {% endblock %}

app/templates/lookup.html

@@ -19,7 +19,9 @@
     {% if error %}
     <div class="box error-box">
         {% if error.value == "NOT_FOUND" %}
-        <p>The remote object was deleted.</p>
+        <p>The remote object is unavailable.</p>
+        {% elif error.value == "UNAUTHORIZED" %}
+        <p>Missing permissions to fetch the remote object.</p>
         {% elif error.value == "TIMEOUT" %}
         <p>Lookup timed out, please try refreshing the page.</p>
         {% else %}

app/templates/notifications.html

@@ -5,10 +5,14 @@
 <title>{{ local_actor.display_name }} - Notifications</title>
 {% endblock %}
 
-{% macro notif_actor_action(notif, text) %}
+{% macro notif_actor_action(notif, text, with_icon=False) %}
     <div class="actor-action">
-        <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.actor.ap_id }}">{{ notif.actor.display_name | clean_html(notif.actor) | safe }}</a> {{ text }}
+        <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.actor.ap_id }}">
+            {% if with_icon %}{{ utils.display_tiny_actor_icon(notif.actor) }}{% endif %} {{ notif.actor.display_name | clean_html(notif.actor) | safe }}</a> {{ text }}
         <span title="{{ notif.created_at.isoformat() }}">{{ notif.created_at | timeago }}</span>
+        {% if notif.is_new %}
+            <span class="new">new</span>
+        {% endif %}
     </div>
 {% endmacro %}
 
@@ -35,17 +39,37 @@
             {%- elif notif.notification_type.value == "follow_request_rejected" %}
                 {{ notif_actor_action(notif, "rejected your follow request") }}
                 {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "blocked" %}
+                {{ notif_actor_action(notif, "blocked you") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "unblocked" %}
+                {{ notif_actor_action(notif, "unblocked you") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "block" %}
+                {{ notif_actor_action(notif, "was blocked") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {% elif notif.notification_type.value == "unblock" %}
+                {{ notif_actor_action(notif, "was unblocked") }}
+                {{ utils.display_actor(notif.actor, actors_metadata) }}
+            {%- elif notif.notification_type.value == "move" and notif.inbox_object %}
+                {# for move notif, the actor is the target and the inbox object the Move activity #}
+                <div class="actor-action">
+                    <a href="{{ url_for("admin_profile") }}?actor_id={{ notif.inbox_object.actor.ap_id }}">
+                        {{ utils.display_tiny_actor_icon(notif.inbox_object.actor) }} {{ notif.inbox_object.actor.display_name | clean_html(notif.inbox_object.actor) | safe }}</a> has moved to
+                    <span title="{{ notif.created_at.isoformat() }}">{{ notif.created_at | timeago }}</span>
+                </div>
+                {{ utils.display_actor(notif.actor) }}
             {% elif notif.notification_type.value == "like" %}
-                {{ notif_actor_action(notif, "liked a post") }}
+                {{ notif_actor_action(notif, "liked a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
            {% elif notif.notification_type.value == "undo_like" %}
-                {{ notif_actor_action(notif, "unliked a post") }}
+                {{ notif_actor_action(notif, "unliked a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "announce" %}
-                {{ notif_actor_action(notif, "shared a post") }}
+                {{ notif_actor_action(notif, "shared a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
            {% elif notif.notification_type.value == "undo_announce" %}
-                {{ notif_actor_action(notif, "unshared a post") }}
+                {{ notif_actor_action(notif, "unshared a post", with_icon=True) }}
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "mention" %}
                 {{ notif_actor_action(notif, "mentioned you") }}
@@ -57,7 +81,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "updated_webmention" %}
@@ -67,7 +91,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% elif notif.notification_type.value == "deleted_webmention" %}
@@ -77,7 +101,7 @@
                     {% if facepile_item %}
                         <a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
                     {% endif %}
-                    <a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
+                    <a class="bold" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
                 </div>
                 {{ utils.display_object(notif.outbox_object) }}
             {% else %}
@@ -88,4 +112,15 @@
     </div>
     {%- endfor %}
     </div>
+
+{% if next_cursor %}
+<div class="box">
+    <p>
+        <a href="{{ request.url._path }}?cursor={{ next_cursor }}">
+            See more{% if more_unread_count %} ({{ more_unread_count }} unread left){% endif %}
+        </a>
+    </p>
+</div>
+{% endif %}
+
 {% endblock %}

app/templates/object.html

@@ -3,7 +3,11 @@
 
 {% block head %}
 {% if outbox_object %}
-{% set excerpt = outbox_object.content | html2text | trim | truncate(50) %}
+{% if outbox_object.content %}
+    {% set excerpt = outbox_object.content | html2text | trim | truncate(50) %}
+{% else %}
+    {% set excerpt = outbox_object.summary | html2text | trim | truncate(50) %}
+{% endif %}
 <title>{% if outbox_object.name %}{{ outbox_object.name }}{% else %}{{ local_actor.display_name }}: "{{ excerpt }}"{% endif %}</title>
 <link rel="webmention" href="{{ url_for("webmention_endpoint") }}">
 <link rel="alternate" href="{{ request.url }}" type="application/activity+json">
@@ -11,7 +15,7 @@
 <meta content="article" property="og:type" />
 <meta content="{{ outbox_object.url }}" property="og:url" />
 <meta content="{{ local_actor.display_name }}'s microblog" property="og:site_name" />
-<meta content="{% if outbox_object.name %}{{ name }}{% else %}Note{% endif %}" property="og:title" />
+<meta content="{% if outbox_object.name %}{{ outbox_object.name }}{% else %}Note{% endif %}" property="og:title" />
 <meta content="{{ excerpt }}" property="og:description" />
 <meta content="{{ local_actor.icon_url }}" property="og:image" />
 <meta content="summary" property="twitter:card" />
@@ -27,9 +31,16 @@
 {% macro display_replies_tree(replies_tree_node) %}
 
 {% if replies_tree_node.is_requested %}
-{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True) }}
+{{ utils.display_object(replies_tree_node.ap_object, likes=likes, shares=shares, webmentions=webmentions, expanded=not replies_tree_node.is_root, is_object_page=True, is_h_entry=False) }}
 {% else %}
-    {{ utils.display_object(replies_tree_node.ap_object) }}
+    {% if replies_tree_node.wm_reply %}
+        {# u-comment h-cite is displayed by default for webmention #}
+        {{ utils.display_webmention_reply(replies_tree_node.wm_reply) }}
+    {% else %}
+        <div class="u-comment h-cite">
+            {{ utils.display_object(replies_tree_node.ap_object, is_h_entry=False) }}
+        </div>
+    {% endif %}
 {% endif %}
 
 {% for child in replies_tree_node.children %}
@@ -38,6 +49,8 @@
 
 {% endmacro %}
 
+<div class="h-entry">
 {{ display_replies_tree(replies_tree) }}
+</div>
 
 {% endblock %}

app/templates/redirect.html

@@ -0,0 +1,15 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ local_actor.display_name }}'s microblog - Redirect</title>
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <p>You are being redirected to: <a href="{{ url }}">{{ url }}</a></p>
+</div>
+
+{% endblock %}

app/templates/redirect_to_remote_instance.html

@@ -0,0 +1,15 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>{{ local_actor.display_name }}'s microblog - Redirect</title>
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <p>You are being redirected to your instance: <a href="{{ url }}">{{ url }}</a></p>
+</div>
+
+{% endblock %}

app/templates/remote_follow.html

@@ -3,6 +3,7 @@
 
 {% block head %}
 <title>Remote follow {{ local_actor.display_name }}</title>
+<meta name="robots" content="noindex, nofollow">
 {% endblock %}
 
 {% block content %}

app/templates/remote_interact.html

@@ -0,0 +1,27 @@
+{%- import "utils.html" as utils with context -%}
+{% extends "layout.html" %}
+
+{% block head %}
+<title>Interact from your instance</title>
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
+
+{% block content %}
+{% include "header.html" %}
+
+<div class="box">
+    <h2>Interact with this object</h2>
+</div>
+
+{{ utils.display_object(outbox_object) }}
+
+<div class="box">
+    <form class="form" action="{{ url_for("post_remote_interaction") }}" method="POST">
+        {{ utils.embed_csrf_token() }}
+        <input type="text" name="profile" placeholder="you@instance.tld" autofocus>
+        <input type="hidden" name="ap_id" value="{{ outbox_object.ap_id }}">
+        <input type="submit" value="interact from your instance">
+    </form>
+</div>
+
+{% endblock %}

app/templates/utils.html

@@ -1,167 +1,254 @@
 {% macro embed_csrf_token() %}
+{% block embed_csrf_token scoped %}
 <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+{% endblock %}
 {% endmacro %}
 
 {% macro embed_redirect_url(permalink_id=None) %}
+{% block embed_redirect_url scoped %}
 <input type="hidden" name="redirect_url" value="{{ request.url }}{% if permalink_id %}#{{ permalink_id }}{% endif %}">
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_block_button(actor) %}
+{% block admin_block_button scoped %}
 <form action="{{ request.url_for("admin_actions_block") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="block">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unblock_button(actor) %}
+{% block admin_unblock_button scoped %}
 <form action="{{ request.url_for("admin_actions_unblock") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="unblock">
 </form>
+{% endblock %}
 {% endmacro %}
 
+{% macro admin_hide_shares_button(actor) %}
+{% block admin_hide_shares_button scoped %}
+<form action="{{ request.url_for("admin_actions_hide_announces") }}" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url() }}
+    <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
+    <input type="submit" value="hide shares">
+</form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_show_shares_button(actor) %}
+{% block admin_show_shares_button scoped %}
+<form action="{{ request.url_for("admin_actions_show_announces") }}" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url() }}
+    <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
+    <input type="submit" value="show shares">
+</form>
+{% endblock %}
+{% endmacro %}
+
+
 {% macro admin_follow_button(actor) %}
+{% block admin_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="ap_actor_id" value="{{ actor.ap_id }}">
     <input type="submit" value="follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_accept_incoming_follow_button(notif) %}
+{% block admin_accept_incoming_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_accept_incoming_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="notification_id" value="{{ notif.id }}">
     <input type="submit" value="accept follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_reject_incoming_follow_button(notif) %}
+{% block admin_reject_incoming_follow_button scoped %}
 <form action="{{ request.url_for("admin_actions_reject_incoming_follow") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url() }}
     <input type="hidden" name="notification_id" value="{{ notif.id }}">
     <input type="submit" value="reject follow">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_like_button(ap_object_id, permalink_id) %}
+{% block admin_like_button scoped %}
 <form action="{{ request.url_for("admin_actions_like") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="like">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_bookmark_button(ap_object_id, permalink_id) %}
+{% block admin_bookmark_button scoped %}
 <form action="{{ request.url_for("admin_actions_bookmark") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="bookmark">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unbookmark_button(ap_object_id, permalink_id) %}
+{% block admin_unbookmark_button scoped %}
 <form action="{{ request.url_for("admin_actions_unbookmark") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="unbookmark">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_pin_button(ap_object_id, permalink_id) %}
+{% block admin_pin_button scoped %}
 <form action="{{ request.url_for("admin_actions_pin") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="pin">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_unpin_button(ap_object_id, permalink_id) %}
+{% block admin_unpin_button scoped %}
 <form action="{{ request.url_for("admin_actions_unpin") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="unpin">
 </form>
+{% endblock %}
 {% endmacro %}
 
-{% macro admin_delete_button(ap_object_id) %}
-<form action="{{ request.url_for("admin_actions_delete") }}" method="POST" onsubmit="return confirm('Do you really want to delete this object?');">
+{% macro admin_delete_button(ap_object) %}
+{% block admin_delete_button scoped %}
+<form action="{{ request.url_for("admin_actions_delete") }}" class="object-delete-form" method="POST">
     {{ embed_csrf_token() }}
-    {{ embed_redirect_url() }}
-    <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
+    <input type="hidden" name="redirect_url" value="{% if request.url.path.endswith("/" + ap_object.public_id) or (request.url.path == "/admin/object" and request.query_params.ap_id.endswith("/" + ap_object.public_id)) %}{{ request.base_url}}{% else %}{{ request.url }}{% endif %}">
+    <input type="hidden" name="ap_object_id" value="{{ ap_object.ap_id }}">
     <input type="submit" value="delete">
 </form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_force_delete_button(ap_object_id, permalink_id=None) %}
+{% block admin_force_delete_button scoped %}
+<form action="{{ request.url_for("admin_actions_force_delete") }}" class="object-delete-form" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url(permalink_id) }}
+    <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
+    <input type="submit" value="local delete">
+</form>
+{% endblock %}
+{% endmacro %}
+
+{% macro admin_force_delete_webmention_button(webmention_id, permalink_id=None) %}
+{% block admin_force_delete_webmention_button scoped %}
+<form action="{{ request.url_for("admin_actions_force_delete_webmention") }}" class="object-delete-form" method="POST">
+    {{ embed_csrf_token() }}
+    {{ embed_redirect_url(permalink_id) }}
+    <input type="hidden" name="webmention_id" value="{{ webmention_id }}">
+    <input type="submit" value="local delete">
+</form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_announce_button(ap_object_id, permalink_id=None) %}
+{% block admin_announce_button scoped %}
 <form action="{{ request.url_for("admin_actions_announce") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="share">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_undo_button(ap_object_id, action="undo", permalink_id=None) %}
+{% block admin_undo_button scoped %}
 <form action="{{ request.url_for("admin_actions_undo") }}" method="POST">
     {{ embed_csrf_token() }}
     {{ embed_redirect_url(permalink_id) }}
     <input type="hidden" name="ap_object_id" value="{{ ap_object_id }}">
     <input type="submit" value="{{ action }}">
 </form>
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_reply_button(ap_object_id) %}
-<form action="/admin/new"  method="GET">
+{% block admin_reply_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="in_reply_to" value="{{ ap_object_id }}">
 <button type="submit">reply</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_dm_button(actor_handle) %}
-<form action="/admin/new"  method="GET">
+{% block admin_dm_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="with_content" value="{{ actor_handle }}">
 <input type="hidden" name="with_visibility" value="DIRECT">
 <button type="submit">direct message</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro admin_mention_button(actor_handle) %}
-<form action="/admin/new"  method="GET">
+{% block admin_mention_button scoped %}
+<form action="{{ BASE_URL }}/admin/new"  method="GET">
 <input type="hidden" name="with_content" value="{{ actor_handle }}">
 <button type="submit">mention</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 
 
 {% macro admin_profile_button(ap_actor_id) %}
+{% block admin_profile_button scoped %}
 <form action="{{ url_for("admin_profile") }}"  method="GET">
 <input type="hidden" name="actor_id" value="{{ ap_actor_id }}">
 <button type="submit">profile</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
-{% macro admin_expand_button(ap_object_id) %}
+{% macro admin_expand_button(ap_object) %}
+{% block admin_expand_button scoped %}
+{# TODO turn these into a regular link and append permalink ID if it's a reply #}
 <form action="{{ url_for("admin_object") }}"  method="GET">
-<input type="hidden" name="ap_id" value="{{ ap_object_id }}">
+<input type="hidden" name="ap_id" value="{{ ap_object.ap_id }}">
 <button type="submit">expand</button>
 </form> 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_box_filters(route) %}
+{% block display_box_filters scoped %}
 <nav class="flexbox box">
 <ul>
     <li>Filter by</li>
@@ -178,17 +265,29 @@
 {% endif %}
 </ul>
 </nav>
+{% endblock %}
 {% endmacro %}
 
-{% macro actor_action(inbox_object, text) %}
+{% macro display_tiny_actor_icon(actor) %}
+{% block display_tiny_actor_icon scoped %}
+    <img class="tiny-actor-icon" src="{{ actor.resized_icon_url }}" alt="">
+{% endblock %}
+{% endmacro %}
+
+{% macro actor_action(inbox_object, text, with_icon=False) %}
+{% block actor_action scoped %}
     <div class="actor-action">
-        <a href="{{ url_for("admin_profile") }}?actor_id={{ inbox_object.actor.ap_id }}">{{ inbox_object.actor.display_name | clean_html(inbox_object.actor) | safe }}</a> {{ text }}
+        <a href="{{ url_for("admin_profile") }}?actor_id={{ inbox_object.actor.ap_id }}">
+            {% if with_icon %}{{ display_tiny_actor_icon(inbox_object.actor) }}{% endif %} {{ inbox_object.actor.display_name | clean_html(inbox_object.actor) | safe }}
+        </a> {{ text }}
         <span title="{{ inbox_object.ap_published_at.isoformat() }}">{{ inbox_object.ap_published_at | timeago }}</span>
     </div>
 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_actor(actor, actors_metadata={}, embedded=False, with_details=False, pending_incoming_follow_notif=None) %}
+{% block display_actor scoped %}
 {% set metadata = actors_metadata.get(actor.ap_id) %}
 
 {% if not embedded %}
@@ -199,7 +298,7 @@
     <div class="icon-box">
         <img src="{{ actor.resized_icon_url }}" alt="{{ actor.display_name }}'s avatar" class="actor-icon u-photo">
     </div>
-    <a href="{{ actor.url }}" class="u-url" style="">
+    <a href="{{ actor.url }}" class="u-url">
         <div><strong>{{ actor.display_name | clean_html(actor) | safe  }}</strong></div>
         <div class="actor-handle p-name">{{ actor.handle }}</div>
     </a>
@@ -209,14 +308,26 @@
 <div>
     <nav class="flexbox actor-metadata">
         <ul>
+            {% if metadata.has_blocked_local_actor %}
+                <li>blocked you</li>
+            {% endif %}
             {% if metadata.is_following %}
                 <li>already following</li>
                 <li>{{ admin_undo_button(metadata.outbox_follow_ap_id, "unfollow")}}</li>
+                {% if not with_details %}
                     <li>{{ admin_profile_button(actor.ap_id) }}</li>
+                {% endif %}
             {% elif metadata.is_follow_request_sent %}
+                {% if metadata.is_follow_request_rejected %}
+                    <li>follow request rejected</li>
+                    {% if not metadata.has_blocked_local_actor %}
+                        <li>{{ admin_follow_button(actor) }}</li>
+                    {% endif %}
+                {% else %}
                     <li>follow request sent</li>
                     <li>{{ admin_undo_button(metadata.outbox_follow_ap_id, "undo follow") }}</li>
-            {% else %}
+                {% endif %}
+            {% elif not actor.moved_to %}
                 <li>{{ admin_follow_button(actor) }}</li>
             {% endif %}
             {% if metadata.is_follower %}
@@ -224,7 +335,11 @@
                 {% if not metadata.is_following and not with_details %}
                     <li>{{ admin_profile_button(actor.ap_id) }}</li>
                 {% endif %}
-                </li>
+            {% elif actor.is_from_db and not with_details and not metadata.is_following %}
+                <li>{{ admin_profile_button(actor.ap_id) }}</li>
+            {% endif %}
+            {% if actor.moved_to %}
+            <li>has moved to {% if metadata.moved_to %}<a href="{{ url_for("admin_profile") }}?actor_id={{ actor.moved_to }}">{{ metadata.moved_to.handle }}</a>{% else %}<a href="{{ url_for("get_lookup") }}?query={{ actor.moved_to }}">{{ actor.moved_to }}</a>{% endif %}</li>
             {% endif %}
             {% if actor.is_from_db %}
                 {% if actor.is_blocked %}
@@ -250,6 +365,14 @@
                     <li>rejected</li>
                 {% endif %}
             {% endif %}
+            {% if actor.are_announces_hidden_from_stream %}
+                <li>{{ admin_show_shares_button(actor) }}</li>
+            {% else %}
+                <li>{{ admin_hide_shares_button(actor) }}</li>
+            {% endif %}
+            {% if with_details %}
+                <li><a href="{{ actor.url }}" class="label-btn">remote profile</a></li>
+            {% endif %}
         </ul>
     </nav>
 </div>
@@ -280,58 +403,74 @@
 </div>
 {% endif %}
 
+{% endblock %}
 {% endmacro %}
 
 {% macro display_og_meta(object) %}
+{% block display_og_meta scoped %}
 {% if object.og_meta %}
-{% for og_meta in object.og_meta %}
-<div class="activity-og-meta" style="display:flex;column-gap: 20px;margin:20px 0;">
+{% for og_meta in object.og_meta[:1] %}
+<div class="activity-og-meta">
     {% if og_meta.image %}
     <div>
-        <img src="{{ og_meta.image | media_proxy_url }}" style="max-width:200px;max-height:100px;">
+        <img src="{{ og_meta.image | media_proxy_url }}">
     </div>
     {% endif %}
     <div>
         <a href="{{ og_meta.url | privacy_replace_url }}">{{ og_meta.title }}</a>
         {% if og_meta.site_name %}
-            <small style="display:block;">{{ og_meta.site_name }}</small>
+            <small>{{ og_meta.site_name }}</small>
         {% endif %}
     </div>
 </div>
 {% endfor %}
 {% endif %}
+{% endblock %}
 {% endmacro %}
 
 
 {% macro display_attachments(object) %}
+{% block display_attachments scoped %}
 
     {% for attachment in object.attachments %}
+    {% if attachment.type != "PropertyValue" %}
+    {% set orientation = "unknown" %}
+    {% if attachment.width %}
+        {% set orientation = "portrait" if attachment.width < attachment.height else "landscape" %}
+    {% endif %}
     {% if object.sensitive and (attachment.type == "Image" or (attachment | has_media_type("image")) or attachment.type == "Video" or (attachment | has_media_type("video"))) %}
-    <div>
-        <label for="{{attachment.proxied_url}}" class="label-btn" style="display:inline-block;">show/hide sensitive content</label>
+    <div class="attachment-wrapper">
+        <label for="{{attachment.proxied_url}}" class="label-btn show-hide-sensitive-btn">show/hide sensitive content</label>
         <div>
         <div class="sensitive-attachment">
         <input class="sensitive-attachment-state" type="checkbox" id="{{attachment.proxied_url}}" aria-hidden="true">
-        <div class="sensitive-attachment-box">
+        <div class="sensitive-attachment-box attachment-orientation-{{orientation}}">
             <div></div>
     {% else %}
-            <div style="margin-top:20px;">
+            <div class="attachment-item attachment-orientation-{{orientation}}">
     {% endif %}
 
     {% if attachment.type == "Image" or (attachment | has_media_type("image")) %}
         {% if attachment.url not in object.inlined_images %}
-            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment" style="margin:0;">
+        <a class="media-link" href="{{ attachment.proxied_url }}" target="_blank">
+            <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} title="{{ attachment.name }}" alt="{{ attachment.name }}"{% endif %} class="attachment u-photo">
+        </a>
         {% endif %}
     {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %}
-    <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %}></video>
+    <div class="video-wrapper">
+    <video controls preload="metadata" src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="u-video"></video>
+    <div class="video-gif-overlay">GIF</div>
+    </div>
     {% elif attachment.type == "Audio" or (attachment | has_media_type("audio")) %}
-    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} style="width:480px;" class="attachment"></audio>
+    <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} class="attachment u-audio"></audio>
     {% elif attachment.type == "Link" %}
-        <a href="{{ attachment.url }}" class="attachment" style="display:inline-block;margin-bottom: 15px;">{{ attachment.url }}</a>
+    <a href="{{ attachment.url }}" class="attachment">{{ attachment.url | truncate(64, True) }}</a> ({{ attachment.mimetype}})
     {% else %}
-    <a href="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="attachment">{{ attachment.url }}</a>
+    <a href="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.url }}"{% endif %} class="attachment">
+        {% if attachment.name %}{{ attachment.name }}{% else %}{{ attachment.url | truncate(64, True) }}{% endif %}
+    </a> ({{ attachment.mimetype }})
     {% endif %}
-    {% if object.sensitive %}
+    {% if object.sensitive and (attachment.type == "Image" or (attachment | has_media_type("image")) or attachment.type == "Video" or (attachment | has_media_type("video"))) %}
             </div>
         </div>
         </div>
@@ -339,13 +478,60 @@
     {% else %}
         </div>
     {% endif %}
+    {% endif %}
     {% endfor %}
+{% endblock %}
 {% endmacro %}
 
-{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False) %}
+{% macro display_webmention_reply(wm_reply) %}
+{% block display_webmention_reply scoped %}
+
+<div class="ap-object u-comment h-cite">
+<div class="actor-box h-card p-author">
+    <div class="icon-box">
+        <img src="{{ wm_reply.face.picture_url }}" alt="{{ wm_reply.face.name }}'s avatar" class="actor-icon u-photo">
+    </div>
+    <a href="{{ wm_reply.face.url }}" class="u-url">
+        <div><strong class="p-name">{{ wm_reply.face.name | clean_html_wm | safe }}</strong></div>
+        <div class="actor-handle">{{ wm_reply.face.url | truncate(64, True) }}</div>
+    </a>
+</div>
+
+    <p class="in-reply-to">in reply to <a href="{{ wm_reply.in_reply_to }}" title="{{ wm_reply.in_reply_to }}" rel="nofollow">
+        this object
+    </a></p>
+
+    <div class="obj-content margin-top-20">
+    <div class="e-content">
+        {{ wm_reply.content | clean_html_wm | safe }}
+    </div>
+    </div>
+
+    <nav class="flexbox activity-bar margin-top-20">
+    <ul>
+        <li>
+            <div><a href="{{ wm_reply.url }}" rel="nofollow" class="object-permalink u-url u-uid">permalink</a></div>
+        </li>
+        <li>
+            <time class="dt-published" datetime="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}" title="{{ wm_reply.published_at.replace(microsecond=0).isoformat() }}">{{ wm_reply.published_at | timeago }}</time>
+        </li>
+        {% if is_admin %}
+            <li>
+                {{ admin_force_delete_webmention_button(wm_reply.webmention_id) }}
+            </li>
+        {% endif %}
+    </ul>
+    </nav>
+</div>
+
+{% endblock %}
+{% endmacro %}
+
+{% macro display_object(object, likes=[], shares=[], webmentions=[], expanded=False, actors_metadata={}, is_object_page=False, is_h_entry=True) %}
+{% block display_object scoped %}
 {% set is_article_mode = object.is_from_outbox and object.ap_type == "Article" and is_object_page %}
-{% if object.ap_type in ["Note", "Article", "Video", "Page", "Question"] %}
-<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}h-entry" id="{{ object.permalink_id }}">
+{% if object.ap_type in ["Note", "Article", "Video", "Page", "Question", "Event"] %}
+<div class="ap-object {% if expanded %}ap-object-expanded {% endif %}{% if is_h_entry %}h-entry{% endif %}" id="{{ object.permalink_id }}">
 
     {% if is_article_mode %}
     <data class="h-card">
@@ -358,13 +544,35 @@
     {% endif %}
 
     {% if object.in_reply_to %}
-        <a href="{% if is_admin %}{{ url_for("get_lookup") }}?query={% endif %}{{ object.in_reply_to }}" title="{{ object.in_reply_to }}" class="in-reply-to" rel="nofollow">
-            in reply to {{ object.in_reply_to|truncate(64, True) }}
-        </a>
+    <p class="in-reply-to">in reply to <a href="{% if is_admin and object.is_in_reply_to_from_inbox %}{{ url_for("get_lookup") }}?query={% endif %}{{ object.in_reply_to }}" title="{{ object.in_reply_to }}" rel="nofollow">
+            this object
+        </a></p>
     {% endif %}
 
-    {% if object.ap_type == "Article" %}
-        <h2 class="p-name" style="margin-top:0;">{{ object.name }}</h2>
+    {% if object.ap_type in ["Article", "Event"] %}
+        <h2 class="p-name no-margin-top">{{ object.name }}</h2>
+    {% endif %}
+
+    {% if object.ap_type == "Event" %}
+        {% if object.ap_object.get("endTime") and object.ap_object.get("startTime") %}
+            <p>On {{ object.ap_object.startTime | parse_datetime | format_date }}
+            (ends {{ object.ap_object.endTime | parse_datetime | format_date }})</p>
+        {% endif %}
+    {% endif %}
+
+    {% if object.ap_object.get("location") %}
+        {% set loc = object.ap_object.get("location") %}
+        {% if loc.type == "Place" and loc.latitude and loc.longitude %}
+        <div class="ap-place">
+            <h3>Location</h3>
+            {% if loc.name %}{{ loc.name }}{% endif %}
+            <span class="h-geo">
+                <data class="p-latitude" value="{{ loc.latitude}}"></data>
+                <data class="p-longitude" value="{{ loc.longitude }}"></data>
+                <a href="https://www.openstreetmap.org/?mlat={{ loc.latitude }}&mlon={{ loc.longitude }}#map=16/{{loc.latitude}}/{{loc.longitude}}">{{loc.latitude}},{{loc.longitude}}</a>
+            </span>
+        </div>
+        {% endif %}
     {% endif %}
 
     {% if is_article_mode %}
@@ -372,12 +580,13 @@
     {% endif %}
 
     {% if object.summary %}
-    <div class="show-more-wrapper">
+    <details class="show-more-wrapper">
+        <summary>
             <div class="p-summary">
                 <p>{{ object.summary | clean_html(object) | safe }}</p>
             </div>
-        <label for="show-more-{{ object.permalink_id }}" class="show-more-btn">show/hide more</label>
-        <input class="show-more-state" type="checkbox" aria-hidden="true" id="show-more-{{ object.permalink_id }}" checked>
+            <span class="show-more-btn" aria-hidden="true"></span>
+        </summary>
     {% endif %}
         <div class="obj-content">
         <div class="e-content">
@@ -394,11 +603,11 @@
         {% endif %}
 
         {% if object.poll_items %}
-            <ul style="list-style-type: none;padding:0;">
+            <ul class="poll-items">
             {% for item in object.poll_items %}
-            <li style="display:block;">
+            <li>
                 {% set pct = item | poll_item_pct(object.poll_voters_count) %}
-                <p style="margin:20px 0 10px 0;">
+                <p>
                     {% if can_vote %}
                     <input type="{% if object.is_one_of_poll %}radio{% else %}checkbox{% endif %}" name="name" value="{{ item.name }}" id="{{object.permalink_id}}-{{item.name}}">
                     <label for="{{object.permalink_id}}-{{item.name}}">
@@ -407,17 +616,17 @@
                     {{ item.name | clean_html(object) | safe }}
 
                     {% if object.voted_for_answers and item.name in object.voted_for_answers %}
-                    <span class="muted" style="padding-left:20px;">you voted for this answer</span>
+                    <span class="muted poll-vote">you voted for this answer</span>
                     {% endif %}
 
                     {% if can_vote %}
                     </label>
                     {% endif %}
 
-                    <span style="float:right;">{{ pct }}% <span class="muted">({{ item.replies.totalItems }} votes)</span></span>
+                    <span class="float-right">{{ pct }}% <span class="muted">({{ item.replies.totalItems }} votes)</span></span>
                 </p>
                 <svg class="poll-bar">
-                    <line x1="0" y1="10px" x2="{{ pct or 1 }}%" y2="10px" style="stroke-width: 20px;"></line>
+                    <line x1="0" y1="10px" x2="{{ pct or 1 }}%" y2="10px"></line>
                 </svg>
             </li>
             {% endfor %}
@@ -431,16 +640,17 @@
             </form>
         {% endif %}
 
-        {{ display_og_meta(object) }}
 
     {% endif %}
 
+        {{ display_og_meta(object) }}
+
         </div>
     {% if object.summary %}
-    </div>
+    </details>
     {% endif %}
 
-    <div class="activity-attachment" style="margin-bottom:20px;">
+    <div class="activity-attachment">
     {{ display_attachments(object) }}
     </div>
 
@@ -449,6 +659,16 @@
         <li>
             <div><a href="{{ object.url }}"{% if object.is_from_inbox %} rel="nofollow"{% endif %} class="object-permalink u-url u-uid">permalink</a></div>
         </li>
+
+        {% if object.is_from_outbox and is_object_page and not is_admin and not request.url.path.startswith("/remote_interaction") %}
+            <li>
+                <a class="label-btn" href="{{ request.url_for("remote_interaction") }}?ap_id={{ object.ap_id }}">
+                    interact from your instance
+                </a>
+            </li>
+        {% endif %}
+
+
         {% if not is_article_mode %}
             <li>
                 <time class="dt-published" datetime="{{ object.ap_published_at.replace(microsecond=0).isoformat() }}" title="{{ object.ap_published_at.replace(microsecond=0).isoformat() }}">{{ object.ap_published_at | timeago }}</time>
@@ -496,7 +716,7 @@
 
         {% if (object.is_from_outbox or is_admin) and object.replies_count %}
         <li>
-            <a href="{% if is_admin and not object.is_from_outbox %}{{ url_for("admin_object") }}?ap_id={{ object.ap_id }}{% else %}{{ object.url }}{% endif %}"><strong>{{ object.replies_count }}</strong> repl{{ object.replies_count | pluralize("y", "ies") }}</a>
+            <a href="{% if is_admin and not object.is_from_outbox %}{{ url_for("admin_object") }}?ap_id={{ object.ap_id }}{% if object.in_reply_to %}#{{ object.permalink_id }}{% endif %}{% else %}{{ object.url }}{% endif %}"><strong>{{ object.replies_count }}</strong> repl{{ object.replies_count | pluralize("y", "ies") }}</a>
         </li>
         {% endif %}
 
@@ -508,7 +728,7 @@
     <ul>
         {% if object.is_from_outbox %}
             <li>
-                {{ admin_delete_button(object.ap_id) }}
+                {{ admin_delete_button(object) }}
             </li>
 
             <li>
@@ -544,7 +764,7 @@
             {% if object.visibility in [visibility_enum.PUBLIC, visibility_enum.UNLISTED] %}
             <li>
                 {% if object.announced_via_outbox_object_ap_id %}
-                    {{ admin_undo_button(object.liked_via_outbox_object_ap_id, "unshare") }}
+                    {{ admin_undo_button(object.announced_via_outbox_object_ap_id, "unshare") }}
                 {% else %}
                     {{ admin_announce_button(object.ap_id, permalink_id=object.permalink_id) }}
                 {% endif %}
@@ -559,7 +779,12 @@
         {% endif %}
         {% if object.is_from_inbox or object.is_from_outbox %}
             <li>
-                {{ admin_expand_button(object.ap_id) }}
+                {{ admin_expand_button(object) }}
+            </li>
+        {% endif %}
+        {% if object.is_from_inbox and not object.announced_via_outbox_object_ap_id and object.is_local_reply %}
+            <li>
+                {{ admin_force_delete_button(object.ap_id) }}
             </li>
         {% endif %}
     </ul>
@@ -568,17 +793,17 @@
 
 
     {% if likes or shares or webmentions %}
-    <div style="display: flex;column-gap: 20px;flex-wrap: wrap;margin-top:20px;">
+    <div class="public-interactions">
         {% if likes %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Likes
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+            <div class="interactions-block">Likes
+                <div class="facepile-wrapper">
                 {% for like in likes %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ like.actor.ap_id }}{% else %}{{ like.actor.url }}{% endif %}" title="{{ like.actor.handle }}" style="height:50px;" rel="noreferrer">
-                        <img src="{{ like.actor.resized_icon_url }}" alt="{{ like.actor.handle}}" style="max-width:50px;">
+                    <a href="{% if is_admin and like.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ like.ap_actor_id }}{% else %}{{ like.url }}{% endif %}" title="{{ like.name }}" rel="noreferrer">
+                        <img src="{{ like.picture_url }}" alt="{{ like.name }}">
                     </a>
                 {% endfor %}
                 {% if object.likes_count > likes | length %}
-                    <div style="display:inline-block;align-self:center;">
+                    <div class="and-x-more">
                         and {{ object.likes_count - likes | length }} more.
                     </div>
                 {% endif %}
@@ -587,15 +812,15 @@
         {% endif %}
 
         {% if shares %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Shares
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+            <div class="interactions-block">Shares
+                <div class="facepile-wrapper">
                 {% for share in shares %}
-                    <a href="{% if is_admin %}{{ url_for("admin_profile") }}?actor_id={{ share.actor.ap_id }}{% else %}{{ share.actor.url }}{% endif %}" title="{{ share.actor.handle }}" style="height:50px;" rel="noreferrer">
-                        <img src="{{ share.actor.resized_icon_url }}" alt="{{ share.actor.handle}}" style="max-width:50px;">
+                    <a href="{% if is_admin and share.ap_actor_id %}{{ url_for("admin_profile") }}?actor_id={{ share.ap_actor_id }}{% else %}{{ share.url }}{% endif %}" title="{{ share.name }}" rel="noreferrer">
+                        <img src="{{ share.picture_url }}" alt="{{ share.name }}">
                     </a>
                 {% endfor %}
                 {% if object.announces_count > shares | length %}
-                    <div style="display:inline-block;align-self:center;">
+                    <div class="and-x-more">
                         and {{ object.announces_count - shares | length }} more.
                     </div>
                 {% endif %}
@@ -604,13 +829,13 @@
         {% endif %}
 
         {% if webmentions %}
-            <div style="flex: 0 1 30%;max-width: 50%;">Webmentions
-                <div style="display: flex;column-gap: 20px;row-gap:20px;flex-wrap: wrap;margin-top:20px;">
+            <div class="interactions-block">Webmentions
+                <div class="facepile-wrapper">
                 {% for webmention in webmentions %}
                     {% set wm = webmention.as_facepile_item %}
                     {% if wm %}
-                        <a href="{{ wm.url }}" title="{{ wm.actor_name }}" style="height:50px;" rel="noreferrer">
-                            <img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}" style="max-width:50px;">
+                        <a href="{{ wm.url }}" title="{{ wm.actor_name }}" rel="noreferrer">
+                            <img src="{{ wm.actor_icon_url | media_proxy_url }}" alt="{{ wm.actor_name }}">
                         </a>
                     {% endif %}
                 {% endfor %}
@@ -624,4 +849,5 @@
 
 </div>
 {% endif %}
+{% endblock %}
 {% endmacro %}

app/uploads.py

@@ -5,6 +5,7 @@ import blurhash  # type: ignore
 from fastapi import UploadFile
 from loguru import logger
 from PIL import Image
+from PIL import ImageOps
 from sqlalchemy import select
 
 from app import activitypub as ap
@@ -45,11 +46,13 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
     width = None
     height = None
 
-    if f.content_type.startswith("image"):
-        image_blurhash = blurhash.encode(f.file, x_components=4, y_components=3)
-        f.file.seek(0)
+    if f.content_type.startswith("image") and not f.content_type == "image/gif":
+        with Image.open(f.file) as _original_image:
+            # Fix image orientation (as we will remove the info from the EXIF
+            # metadata)
+            original_image = ImageOps.exif_transpose(_original_image)
 
-        with Image.open(f.file) as original_image:
+            # Re-creating the image drop the EXIF metadata
             destination_image = Image.new(
                 original_image.mode,
                 original_image.size,
@@ -57,15 +60,18 @@ async def save_upload(db_session: AsyncSession, f: UploadFile) -> models.Upload:
             destination_image.putdata(original_image.getdata())
             destination_image.save(
                 dest_filename,
-                format=original_image.format,
+                format=_original_image.format,  # type: ignore
             )
 
+            with open(dest_filename, "rb") as dest_f:
+                image_blurhash = blurhash.encode(dest_f, x_components=4, y_components=3)
+
             try:
-                width, height = original_image.size
-                original_image.thumbnail((740, 740))
-                original_image.save(
+                width, height = destination_image.size
+                destination_image.thumbnail((740, 740))
+                destination_image.save(
                     UPLOAD_DIR / f"{content_hash}_resized",
-                    format=original_image.format,
+                    format="webp",
                 )
             except Exception:
                 logger.exception(

app/utils/custom_index_handler.py

@@ -0,0 +1,32 @@
+from typing import Any
+from typing import Awaitable
+from typing import Callable
+
+from fastapi import Depends
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+from app.actor import LOCAL_ACTOR
+from app.config import is_activitypub_requested
+from app.database import AsyncSession
+from app.database import get_db_session
+
+_Handler = Callable[[Request, AsyncSession], Awaitable[Any]]
+
+
+def build_custom_index_handler(handler: _Handler) -> _Handler:
+    async def custom_index(
+        request: Request,
+        db_session: AsyncSession = Depends(get_db_session),
+    ) -> Any:
+        # Serve the AP actor if requested
+        if is_activitypub_requested(request):
+            return JSONResponse(
+                LOCAL_ACTOR.ap_actor,
+                media_type="application/activity+json",
+            )
+
+        # Defer to the custom handler
+        return await handler(request, db_session)
+
+    return custom_index

app/utils/emoji.py

@@ -23,6 +23,8 @@ def _load_emojis(root_dir: Path, base_url: str) -> None:
             mt = mimetypes.guess_type(emoji.name)[0]
             if mt and mt.startswith("image/"):
                 name = emoji.name.split(".")[0]
+                if not re.match(EMOJI_REGEX, f":{name}:"):
+                    continue
                 ap_emoji: "RawObject" = {
                     "type": "Emoji",
                     "name": f":{name}:",

app/utils/facepile.py

@@ -0,0 +1,172 @@
+import datetime
+from dataclasses import dataclass
+from datetime import timezone
+from typing import Any
+from typing import Optional
+
+from loguru import logger
+
+from app import media
+from app.models import InboxObject
+from app.models import Webmention
+from app.utils.datetime import parse_isoformat
+from app.utils.url import must_make_abs
+
+
+@dataclass
+class Face:
+    ap_actor_id: str | None
+    url: str
+    name: str
+    picture_url: str
+    created_at: datetime.datetime
+
+    @classmethod
+    def from_inbox_object(cls, like: InboxObject) -> "Face":
+        return cls(
+            ap_actor_id=like.actor.ap_id,
+            url=like.actor.url,  # type: ignore
+            name=like.actor.handle,  # type: ignore
+            picture_url=like.actor.resized_icon_url,
+            created_at=like.created_at,  # type: ignore
+        )
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["Face"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-card":
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=(
+                            must_make_abs(
+                                item["properties"]["url"][0], webmention.source
+                            )
+                            if item["properties"].get("url")
+                            else webmention.source
+                        ),
+                        name=item["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            must_make_abs(
+                                item["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+            elif item["type"][0] == "h-entry":
+                author = item["properties"]["author"][0]
+                try:
+                    return cls(
+                        ap_actor_id=None,
+                        url=webmention.source,
+                        name=author["properties"]["name"][0],
+                        picture_url=media.resized_media_url(
+                            must_make_abs(
+                                author["properties"]["photo"][0], webmention.source
+                            ),  # type: ignore
+                            50,
+                        ),
+                        created_at=webmention.created_at,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None
+
+
+def merge_faces(faces: list[Face]) -> list[Face]:
+    return sorted(
+        faces,
+        key=lambda f: f.created_at,
+        reverse=True,
+    )[:10]
+
+
+def _parse_face(webmention: Webmention, items: list[dict[str, Any]]) -> Face | None:
+    for item in items:
+        if item["type"][0] == "h-card":
+            try:
+                return Face(
+                    ap_actor_id=None,
+                    url=(
+                        must_make_abs(item["properties"]["url"][0], webmention.source)
+                        if item["properties"].get("url")
+                        else webmention.source
+                    ),
+                    name=item["properties"]["name"][0],
+                    picture_url=media.resized_media_url(
+                        must_make_abs(
+                            item["properties"]["photo"][0], webmention.source
+                        ),  # type: ignore
+                        50,
+                    ),
+                    created_at=webmention.created_at,  # type: ignore
+                )
+            except Exception:
+                logger.exception(
+                    f"Failed to build Face for webmention id={webmention.id}"
+                )
+                break
+
+    return None
+
+
+@dataclass
+class WebmentionReply:
+    face: Face
+    content: str
+    url: str
+    published_at: datetime.datetime
+    in_reply_to: str
+    webmention_id: int
+
+    @classmethod
+    def from_webmention(cls, webmention: Webmention) -> Optional["WebmentionReply"]:
+        items = webmention.source_microformats.get("items", [])  # type: ignore
+        for item in items:
+            if item["type"][0] == "h-entry":
+                try:
+                    face = _parse_face(webmention, item["properties"].get("author", []))
+                    if not face:
+                        logger.info(
+                            "Failed to build WebmentionReply/Face for "
+                            f"webmention id={webmention.id}"
+                        )
+                        break
+
+                    if "published" in item["properties"]:
+                        published_at = (
+                            parse_isoformat(item["properties"]["published"][0])
+                            .astimezone(timezone.utc)
+                            .replace(tzinfo=None)
+                        )
+                    else:
+                        published_at = webmention.created_at  # type: ignore
+
+                    return cls(
+                        face=face,
+                        content=item["properties"]["content"][0]["html"],
+                        url=must_make_abs(
+                            item["properties"]["url"][0], webmention.source
+                        ),
+                        published_at=published_at,
+                        in_reply_to=webmention.target,  # type: ignore
+                        webmention_id=webmention.id,  # type: ignore
+                    )
+                except Exception:
+                    logger.exception(
+                        f"Failed to build Face for webmention id={webmention.id}"
+                    )
+                    break
+
+        return None

app/utils/highlight.py

@@ -1,3 +1,5 @@
+import base64
+import hashlib
 from functools import lru_cache
 
 from bs4 import BeautifulSoup  # type: ignore
@@ -11,6 +13,9 @@ from app.config import CODE_HIGHLIGHTING_THEME
 _FORMATTER = HtmlFormatter(style=CODE_HIGHLIGHTING_THEME)
 
 HIGHLIGHT_CSS = _FORMATTER.get_style_defs()
+HIGHLIGHT_CSS_HASH = base64.b64encode(
+    hashlib.sha256(HIGHLIGHT_CSS.encode()).digest()
+).decode()
 
 
 @lru_cache(256)
@@ -27,15 +32,11 @@ def highlight(html: str) -> str:
 
         # If this comes from a microblog.pub instance we may have the language
         # in the class name
-        if "class" in code.attrs and code.attrs["class"][0].startswith("language-"):
+        if "data-microblogpub-lexer" in code.attrs:
             try:
-                lexer = get_lexer_by_name(
-                    code.attrs["class"][0].removeprefix("language-")
-                )
+                lexer = get_lexer_by_name(code.attrs["data-microblogpub-lexer"])
             except Exception:
                 lexer = guess_lexer(code_content)
-        else:
-            lexer = guess_lexer(code_content)
 
             # Replace the code with Pygment output
             # XXX: the HTML escaping causes issue with Python type annotations
@@ -45,5 +46,8 @@ def highlight(html: str) -> str:
                     phighlight(code_content, lexer, _FORMATTER), "html5lib"
                 ).body.next
             )
+        else:
+            code.name = "div"
+            code["class"] = code.get("class", []) + ["highlight"]
 
     return soup.body.encode_contents().decode()

app/utils/indieauth.py

@@ -10,7 +10,7 @@ from app.utils.url import make_abs
 class IndieAuthClient:
     logo: str | None
     name: str
-    url: str
+    url: str | None
 
 
 def _get_prop(props: dict[str, Any], name: str, default=None) -> Any:

app/utils/mastodon.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from loguru import logger
+
+from app.webfinger import get_actor_url
+
+
+def _load_mastodon_following_accounts_csv_file(path: str) -> list[str]:
+    handles = []
+    for line in Path(path).read_text().splitlines()[1:]:
+        handle = line.split(",")[0]
+        handles.append(handle)
+
+    return handles
+
+
+async def get_actor_urls_from_following_accounts_csv_file(
+    path: str,
+) -> list[tuple[str, str]]:
+    actor_urls = []
+    for handle in _load_mastodon_following_accounts_csv_file(path):
+        try:
+            actor_url = await get_actor_url(handle)
+        except Exception:
+            logger.error("Failed to fetch actor URL for {handle=}")
+        else:
+            if actor_url:
+                actor_urls.append((handle, actor_url))
+            else:
+                logger.info(f"No actor URL found for {handle=}")
+
+    return actor_urls

app/utils/opengraph.py

@@ -1,13 +1,18 @@
+import asyncio
 import mimetypes
 import re
+import signal
+from concurrent.futures import TimeoutError
 from typing import Any
 from urllib.parse import urlparse
 
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from loguru import logger
+from pebble import concurrent  # type: ignore
 from pydantic import BaseModel
 
+from app import activitypub as ap
 from app import ap_object
 from app import config
 from app.actor import LOCAL_ACTOR
@@ -27,7 +32,11 @@ class OpenGraphMeta(BaseModel):
     site_name: str
 
 
+@concurrent.process(timeout=5)
 def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
+    # Prevent SIGTERM to bubble up to the worker
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
     soup = BeautifulSoup(html, "html5lib")
     ogs = {
         og.attrs["property"]: og.attrs.get("content")
@@ -36,7 +45,7 @@ def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
     # FIXME some page have no <title>
     raw = {
         "url": url,
-        "title": soup.find("title").text,
+        "title": soup.find("title").text.strip(),
         "image": None,
         "description": None,
         "site_name": urlparse(url).hostname,
@@ -53,9 +62,20 @@ def _scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
         if u := raw.get(maybe_rel):
             raw[maybe_rel] = make_abs(u, url)
 
+            if not is_url_valid(raw[maybe_rel]):
+                logger.info(f"Invalid url {raw[maybe_rel]}")
+                if maybe_rel == "url":
+                    raw["url"] = url
+                elif maybe_rel == "image":
+                    raw["image"] = None
+
     return OpenGraphMeta.parse_obj(raw)
 
 
+def scrap_og_meta(url: str, html: str) -> OpenGraphMeta | None:
+    return _scrap_og_meta(url, html).result()
+
+
 async def external_urls(
     db_session: AsyncSession,
     ro: ap_object.RemoteObject | OutboxObject | InboxObject,
@@ -68,7 +88,12 @@ async def external_urls(
             tags_hrefs.add(tag_href)
         if tag.get("type") == "Mention":
             if tag["href"] != LOCAL_ACTOR.ap_id:
+                try:
                     mentioned_actor = await fetch_actor(db_session, tag["href"])
+                except (ap.FetchError, ap.NotAnObjectError):
+                    tags_hrefs.add(tag["href"])
+                    continue
+
                 tags_hrefs.add(mentioned_actor.url)
                 tags_hrefs.add(mentioned_actor.ap_id)
             else:
@@ -80,6 +105,10 @@ async def external_urls(
         soup = BeautifulSoup(ro.content, "html5lib")
         for link in soup.find_all("a"):
             h = link.get("href")
+            if not h:
+                continue
+
+            try:
                 ph = urlparse(h)
                 mimetype, _ = mimetypes.guess_type(h)
                 if (
@@ -92,6 +121,9 @@ async def external_urls(
                     )
                 ):
                     urls.add(h)
+            except Exception:
+                logger.exception(f"Failed to check {h}")
+                continue
 
     return urls - tags_hrefs
 
@@ -112,7 +144,10 @@ async def _og_meta_from_url(url: str) -> OpenGraphMeta | None:
         return None
 
     try:
-        return _scrap_og_meta(url, resp.text)
+        return scrap_og_meta(url, resp.text)
+    except TimeoutError:
+        logger.info(f"Timed out when scraping OG meta for {url}")
+        return None
     except Exception:
         logger.info(f"Failed to scrap OG meta for {url}")
         return None
@@ -124,9 +159,21 @@ async def og_meta_from_note(
 ) -> list[dict[str, Any]]:
     og_meta = []
     urls = await external_urls(db_session, ro)
+    logger.debug(f"Lookig OG metadata in {urls=}")
     for url in urls:
+        logger.debug(f"Processing {url}")
         try:
-            maybe_og_meta = await _og_meta_from_url(url)
+            maybe_og_meta = None
+            try:
+                maybe_og_meta = await asyncio.wait_for(
+                    _og_meta_from_url(url),
+                    timeout=5,
+                )
+            except asyncio.TimeoutError:
+                logger.info(f"Timing out fetching {url}")
+            except Exception:
+                logger.exception(f"Failed scrap OG meta for {url}")
+
             if maybe_og_meta:
                 og_meta.append(maybe_og_meta.dict())
         except httpx.HTTPError:

app/utils/text.py

@@ -0,0 +1,8 @@
+import re
+import unicodedata
+
+
+def slugify(text: str) -> str:
+    value = unicodedata.normalize("NFKC", text)
+    value = re.sub(r"[^\w\s-]", "", value.lower())
+    return re.sub(r"[-\s]+", "-", value).strip("-_")

app/utils/url.py

@@ -21,6 +21,13 @@ def make_abs(url: str | None, parent: str) -> str | None:
     )
 
 
+def must_make_abs(url: str | None, parent: str) -> str:
+    abs_url = make_abs(url, parent)
+    if not abs_url:
+        raise ValueError("missing URL")
+    return abs_url
+
+
 class InvalidURLError(Exception):
     pass
 
@@ -54,10 +61,14 @@ def is_url_valid(url: str) -> bool:
     if not parsed.hostname or parsed.hostname.lower() in ["localhost"]:
         return False
 
-    if parsed.hostname in BLOCKED_SERVERS:
+    if is_hostname_blocked(parsed.hostname):
         logger.warning(f"{parsed.hostname} is blocked")
         return False
 
+    if parsed.hostname.endswith(".onion"):
+        logger.warning(f"{url} is an onion service")
+        return False
+
     ip_address = _getaddrinfo(
         parsed.hostname, parsed.port or (80 if parsed.scheme == "http" else 443)
     )
@@ -77,3 +88,11 @@ def check_url(url: str) -> None:
         raise InvalidURLError(f'"{url}" is invalid')
 
     return None
+
+
+@functools.lru_cache(maxsize=256)
+def is_hostname_blocked(hostname: str) -> bool:
+    for blocked_hostname in BLOCKED_SERVERS:
+        if hostname == blocked_hostname or hostname.endswith(f".{blocked_hostname}"):
+            return True
+    return False

app/utils/version.py

@@ -0,0 +1,12 @@
+import subprocess
+
+
+def get_version_commit() -> str:
+    try:
+        return (
+            subprocess.check_output(["git", "rev-parse", "--short=8", "v2"])
+            .split()[0]
+            .decode()
+        )
+    except Exception:
+        return "dev"

app/utils/webmentions.py

@@ -24,7 +24,7 @@ async def _discover_webmention_endoint(url: str) -> str | None:
                 follow_redirects=True,
             )
             resp.raise_for_status()
-        except (httpx.HTTPError, httpx.HTTPStatusError):
+        except Exception:
             logger.exception(f"Failed to discover webmention endpoint for {url}")
             return None
 

app/utils/workers.py

@@ -54,15 +54,20 @@ class Worker(Generic[T]):
                 {task, stop_task}, return_when=asyncio.FIRST_COMPLETED
             )
             logger.info(f"Waiting for tasks to finish {done=}/{pending=}")
-            await asyncio.sleep(5)
             tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
             logger.info(f"Cancelling {len(tasks)} tasks")
             [task.cancel() for task in tasks]
 
-        await asyncio.gather(*tasks, return_exceptions=True)
+        try:
+            await asyncio.wait_for(
+                asyncio.gather(*tasks, return_exceptions=True),
+                timeout=15,
+            )
+        except asyncio.TimeoutError:
+            logger.info("Tasks failed to cancel")
 
         logger.info("stopping loop")
 
     async def _shutdown(self, sig: signal.Signals) -> None:
-        logger.info(f"Caught {signal=}")
+        logger.info(f"Caught {sig=}")
         self._stop_event.set()

app/utils/yunohost.py

@@ -6,7 +6,6 @@ from typing import Any
 
 import bcrypt
 import tomli_w
-from markdown import markdown  # type: ignore
 
 from app.key import generate_key
 
@@ -44,7 +43,7 @@ def setup_config_file(
     dat["username"] = username
     dat["admin_password"] = bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
     dat["name"] = name
-    dat["summary"] = markdown(summary)
+    dat["summary"] = summary
     dat["https"] = True
     proto = "https"
     dat["icon_url"] = f'{proto}://{dat["domain"]}/static/nopic.png'

app/webfinger.py

@@ -1,3 +1,4 @@
+import xml.etree.ElementTree as ET
 from typing import Any
 from urllib.parse import urlparse
 
@@ -8,31 +9,85 @@ from app import config
 from app.utils.url import check_url
 
 
+async def get_webfinger_via_host_meta(host: str) -> str | None:
+    resp: httpx.Response | None = None
+    is_404 = False
+    async with httpx.AsyncClient() as client:
+        for i, proto in enumerate({"http", "https"}):
+            try:
+                url = f"{proto}://{host}/.well-known/host-meta"
+                check_url(url)
+                resp = await client.get(
+                    url,
+                    headers={
+                        "User-Agent": config.USER_AGENT,
+                    },
+                    follow_redirects=True,
+                )
+                resp.raise_for_status()
+                break
+            except httpx.HTTPStatusError as http_error:
+                logger.exception("HTTP error")
+                if http_error.response.status_code in [403, 404, 410]:
+                    is_404 = True
+                    continue
+                raise
+            except httpx.HTTPError:
+                logger.exception("req failed")
+                # If we tried https first and the domain is "http only"
+                if i == 0:
+                    continue
+                break
+
+        if is_404:
+            return None
+
+    if resp:
+        tree = ET.fromstring(resp.text)
+        maybe_link = tree.find(
+            "./{http://docs.oasis-open.org/ns/xri/xrd-1.0}Link[@rel='lrdd']"
+        )
+        if maybe_link is not None:
+            return maybe_link.attrib.get("template")
+
+    return None
+
+
 async def webfinger(
     resource: str,
+    webfinger_url: str | None = None,
 ) -> dict[str, Any] | None:  # noqa: C901
     """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL."""
+    resource = resource.strip()
     logger.info(f"performing webfinger resolution for {resource}")
-    protos = ["https", "http"]
+    urls = []
+    host = None
+    if webfinger_url:
+        urls = [webfinger_url]
+    else:
         if resource.startswith("http://"):
-        protos.reverse()
             host = urlparse(resource).netloc
+            url = f"http://{host}/.well-known/webfinger"
         elif resource.startswith("https://"):
             host = urlparse(resource).netloc
+            url = f"https://{host}/.well-known/webfinger"
         else:
+            protos = ["https", "http"]
+            _, host = resource.split("@", 1)
+            urls = [f"{proto}://{host}/.well-known/webfinger" for proto in protos]
+
     if resource.startswith("acct:"):
         resource = resource[5:]
     if resource.startswith("@"):
         resource = resource[1:]
-        _, host = resource.split("@", 1)
     resource = "acct:" + resource
 
     is_404 = False
 
+    resp: httpx.Response | None = None
     async with httpx.AsyncClient() as client:
-        for i, proto in enumerate(protos):
+        for i, url in enumerate(urls):
             try:
-                url = f"{proto}://{host}/.well-known/webfinger"
                 check_url(url)
                 resp = await client.get(
                     url,
@@ -56,10 +111,20 @@ async def webfinger(
                 if i == 0:
                     continue
                 break
+
     if is_404:
+        if not webfinger_url and host:
+            if webfinger_url := (await get_webfinger_via_host_meta(host)):
+                return await webfinger(
+                    resource,
+                    webfinger_url=webfinger_url,
+                )
         return None
 
+    if resp:
         return resp.json()
+    else:
+        return None
 
 
 async def get_remote_follow_template(resource: str) -> str | None:

app/webmentions.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 import httpx
 from bs4 import BeautifulSoup  # type: ignore
 from fastapi import APIRouter
@@ -6,13 +8,21 @@ from fastapi import HTTPException
 from fastapi import Request
 from fastapi.responses import JSONResponse
 from loguru import logger
+from sqlalchemy import func
 from sqlalchemy import select
 
 from app import models
+from app.boxes import _get_outbox_announces_count
+from app.boxes import _get_outbox_likes_count
+from app.boxes import _get_outbox_replies_count
 from app.boxes import get_outbox_object_by_ap_id
+from app.boxes import get_outbox_object_by_slug_and_short_id
+from app.boxes import is_notification_enabled
 from app.database import AsyncSession
 from app.database import get_db_session
 from app.utils import microformats
+from app.utils.facepile import Face
+from app.utils.facepile import WebmentionReply
 from app.utils.url import check_url
 from app.utils.url import is_url_valid
 
@@ -47,6 +57,7 @@ async def webmention_endpoint(
 
         check_url(source)
         check_url(target)
+        parsed_target_url = urlparse(target)
     except Exception:
         logger.exception("Invalid webmention request")
         raise HTTPException(status_code=400, detail="Invalid payload")
@@ -65,6 +76,16 @@ async def webmention_endpoint(
         logger.info("Found existing Webmention, will try to update or delete")
 
     mentioned_object = await get_outbox_object_by_ap_id(db_session, target)
+
+    if not mentioned_object and parsed_target_url.path.startswith("/articles/"):
+        try:
+            _, _, short_id, slug = parsed_target_url.path.split("/")
+            mentioned_object = await get_outbox_object_by_slug_and_short_id(
+                db_session, slug, short_id
+            )
+        except Exception:
+            logger.exception(f"Failed to match {target}")
+
     if not mentioned_object:
         logger.info(f"Invalid target {target=}")
 
@@ -90,9 +111,15 @@ async def webmention_endpoint(
         logger.warning(f"target {target=} not found in source")
         if existing_webmention_in_db:
             logger.info("Deleting existing Webmention")
-            mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
             existing_webmention_in_db.is_deleted = True
+            await db_session.flush()
 
+            # Revert side effects
+            await _handle_webmention_side_effects(
+                db_session, existing_webmention_in_db, mentioned_object
+            )
+
+            if is_notification_enabled(models.NotificationType.DELETED_WEBMENTION):
                 notif = models.Notification(
                     notification_type=models.NotificationType.DELETED_WEBMENTION,
                     outbox_object_id=mentioned_object.id,
@@ -110,11 +137,16 @@ async def webmention_endpoint(
         else:
             return JSONResponse(content={}, status_code=200)
 
+    webmention_type = models.WebmentionType.UNKNOWN
+    webmention: models.Webmention
     if existing_webmention_in_db:
         # Undelete if needed
         existing_webmention_in_db.is_deleted = False
         existing_webmention_in_db.source_microformats = data
+        await db_session.flush()
+        webmention = existing_webmention_in_db
 
+        if is_notification_enabled(models.NotificationType.UPDATED_WEBMENTION):
             notif = models.Notification(
                 notification_type=models.NotificationType.UPDATED_WEBMENTION,
                 outbox_object_id=mentioned_object.id,
@@ -127,10 +159,13 @@ async def webmention_endpoint(
             target=target,
             source_microformats=data,
             outbox_object_id=mentioned_object.id,
+            webmention_type=webmention_type,
         )
         db_session.add(new_webmention)
         await db_session.flush()
+        webmention = new_webmention
 
+        if is_notification_enabled(models.NotificationType.NEW_WEBMENTION):
             notif = models.Notification(
                 notification_type=models.NotificationType.NEW_WEBMENTION,
                 outbox_object_id=mentioned_object.id,
@@ -138,8 +173,60 @@ async def webmention_endpoint(
             )
             db_session.add(notif)
 
-        mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1
+    # Determine the webmention type
+    for item in data.get("items", []):
+        if target in item.get("properties", {}).get(
+            "in-reply-to", []
+        ) and WebmentionReply.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPLY
+            break
+        elif target in item.get("properties", {}).get(
+            "like-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.LIKE
+            break
+        elif target in item.get("properties", {}).get(
+            "repost-of", []
+        ) and Face.from_webmention(webmention):
+            webmention_type = models.WebmentionType.REPOST
+            break
 
+    if webmention_type != models.WebmentionType.UNKNOWN:
+        webmention.webmention_type = webmention_type
+        await db_session.flush()
+
+    # Handle side effect
+    await _handle_webmention_side_effects(db_session, webmention, mentioned_object)
     await db_session.commit()
 
     return JSONResponse(content={}, status_code=200)
+
+
+async def _handle_webmention_side_effects(
+    db_session: AsyncSession,
+    webmention: models.Webmention,
+    mentioned_object: models.OutboxObject,
+) -> None:
+    if webmention.webmention_type == models.WebmentionType.UNKNOWN:
+        # TODO: recount everything
+        mentioned_object.webmentions_count = await db_session.scalar(
+            select(func.count(models.Webmention.id)).where(
+                models.Webmention.is_deleted.is_(False),
+                models.Webmention.outbox_object_id == mentioned_object.id,
+                models.Webmention.webmention_type == models.WebmentionType.UNKNOWN,
+            )
+        )
+    elif webmention.webmention_type == models.WebmentionType.LIKE:
+        mentioned_object.likes_count = await _get_outbox_likes_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPOST:
+        mentioned_object.announces_count = await _get_outbox_announces_count(
+            db_session, mentioned_object
+        )
+    elif webmention.webmention_type == models.WebmentionType.REPLY:
+        mentioned_object.replies_count = await _get_outbox_replies_count(
+            db_session, mentioned_object
+        )
+    else:
+        raise ValueError(f"Unhandled {webmention.webmention_type} webmention")

data/templates/app

@@ -0,0 +1 @@
+../../app/templates/

data/tests.toml

@@ -5,6 +5,7 @@ admin_password = "$2b$12$OwCyZM33uXQUVrChgER.h.qgFJ4fBp6tdFwArR3Lm1LV8NgMvIxVa"
 name = "test"
 summary = "<p>Hello</p>"
 https = false
+id = "http://localhost:8000"
 icon_url = "https://localhost:8000/static/nopic.png"
 secret = "1dd4079e0474d1a519052b8fe3cb5fa6"
 debug = true

docs/developer_guide.md

@@ -1,6 +1,6 @@
 # Developer's guide
 
-This guide assume you have some knoweldge of [ActivityPub](https://activitypub.rocks/).
+This guide assumes you have some knowledge of [ActivityPub](https://activitypub.rocks/).
 
 [TOC]
 
@@ -10,12 +10,13 @@ Microblog.pub is a "modern" Python application with "old-school" server-rendered
 
  - [Poetry](https://python-poetry.org/) is used for dependency management.
  - Most of the code is asynchronous, using [asyncio](https://docs.python.org/3/library/asyncio.html).
- - SQLite3 is the default database.
+ - SQLite3 for data storage
 
-The server has 2 components:
+The server has 3 components:
 
  - The web server (powered by [FastAPI](https://fastapi.tiangolo.com/) and [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) templates)
- - An additional process that takes care of sending "outgoing actities" 
+ - One process that takes care of sending "outgoing activities" 
+ - One process that takes care of processing "incoming activities" 
 
 ### Tasks
 
@@ -29,7 +30,7 @@ inv -l
 
 ### Media storage
 
-The uploads are stored in the `data/` directory, using a simple content-addressed storage (file contents hash is the name of the store BLOB).
+The uploads are stored in the `data/` directory, using a simple content-addressed storage system (file contents hash is BLOB filename).
 Files metadata are stored in the database.
 
 ## Installation
@@ -57,3 +58,24 @@ And check out the result by starting a static server using Python standard libra
 cd docs/dist
 python -m http.server 8001
 ```
+
+## Contributing
+
+Contributions/patches are welcome, but please start a discussion in a [ticket](https://todo.sr.ht/~tsileo/microblog.pub) or a [thread in the mailing list](https://lists.sr.ht/~tsileo/microblog.pub-devel) before working on anything consequent.
+
+### Patches
+
+Please ensure your code passes the code quality checks:
+
+```bash
+inv autoformat
+inv lint
+```
+
+And that the tests suite is passing:
+
+```bash
+inv tests
+```
+
+Please also consider adding new test cases if needed.

docs/install.md

@@ -11,7 +11,7 @@ For now, there's no image published on Docker Hub, this means you will have to b
 Clone the repository, replace `you-domain.tld` by your own domain.
 
 Note that if you want to serve static assets via your reverse proxy (like nginx), clone it in a place
-where accessible by your reverse proxy user.
+where it is accessible by your reverse proxy user.
 
 ```bash
 git clone https://git.sr.ht/~tsileo/microblog.pub your-domain.tld
@@ -55,6 +55,12 @@ docker compose stop
 docker compose up -d
 ```
 
+As you probably already know, Docker can (and will) eat a lot of disk space, when updating you should [prune old images](https://docs.docker.com/config/pruning/#prune-images) from time to time:
+
+```bash
+docker image prune -a --filter "until=24h"
+```
+
 ## Python developer edition
 
 Assuming you have a working **Python 3.10+** environment. 
@@ -83,6 +89,12 @@ Setup config.
 poetry run inv configuration-wizard
 ```
 
+Setup the database.
+
+```bash
+poetry run inv migrate-db
+```
+
 Grab your virtualenv path.
 
 ```bash
@@ -99,7 +111,7 @@ Setup a reverse proxy (see the next section).
 
 ### Updating 
 
-To update microblogpub locally, pull the remote changes and run the `update` task to regeneratee the CSS and run any DB migrations.
+To update microblogpub locally, pull the remote changes and run the `update` task to regenerate the CSS and run any DB migrations.
 
 ```bash
 git pull
@@ -130,6 +142,11 @@ server {
     # [...]
 }
 
+# This should be outside the `server` block
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
 ```
 
 Optionally, you can serve static files using NGINX directly, with an additional `location` block.
@@ -147,8 +164,99 @@ server {
        # path for static files
        rewrite ^/static/(.*) /$1 break;
        root /path/to/your-domain.tld/app/static/;
+       expires 1y;
     }
 
     # [...]
 }
 ```
+
+### NGINX config tips
+
+Enable HTTP2 (which is disabled by default):
+
+```nginx
+server {
+    # [...]
+    listen [::]:443 ssl http2;
+}
+```
+
+Tweak `/etc/nginx/nginx.conf` and add gzip compression for ActivityPub responses:
+
+```nginx
+http {
+    # [...]
+    gzip_types text/plain text/css application/json application/javascript application/activity+json application/octet-stream;
+}
+```
+
+
+## (Advanced) Running on a subdomain
+
+It is possible to run microblogpub on a subdomain (`sub.domain.tld`) while being reachable from the root root domain (`domain.tld`) using the `name@domain.tld` handle.
+
+This requires forwarding/proxying requests from the root domain to the subdomain, for example using NGINX:
+
+```nginx
+location /.well-known/webfinger {
+  add_header Access-Control-Allow-Origin '*';
+  return 301 https://sub.domain.tld$request_uri;
+}
+```
+
+And updating `data/profile.toml` to specify the root domain as the webfinger domain:
+
+```toml
+webfinger_domain = "domain.tld"
+```
+
+Once configured correctly, people will be able to follow you using `name@domain.tld`, while using `sub.domain.tld` for the web interface.
+
+
+## (Advanced) Running from subpath
+
+It is possible to configure microblogpub to run from subpath.
+To achieve this, do the following configuration _between_ config and start steps.
+i.e. _after_ you run `make config` or `poetry run inv configuration-wizard`,
+but _before_ you run `docker compose up` or `poetry run supervisord`.
+Changing this settings on an instance which has some posts or was seen by other instances will likely break links to these posts or federation (i.e. links to your instance, posts and profile from other instances).
+
+The following steps will explain how to configure instance to be available at `https://example.com/subdir`.
+Change them to your actual domain and subdir.
+
+* Edit `data/profile.toml` file, add this line:
+
+		id = "https://example.com/subdir"
+
+* Edit `misc/*-supervisord.conf` file which is relevant to you (it depends on how you start microblogpub - if in doubt, do the same change in all of them) - in `[program:uvicorn]` section, in the line which starts with `command`, add this argument at the very end: ` --root-path /subdir`
+
+Above two steps are enough to configure microblogpub.
+Next, you also need to configure reverse proxy.
+It might slightly differ if you plan to have other services running on the same domain, but for [NGINX config shown above](#reverse-proxy), the following changes are enough:
+
+* Add subdir to location, so location block starts like this:
+
+		location /subdir {
+
+* Add `/` at the end of `proxy_pass` directive, like this:
+
+		proxy_pass http://localhost:8000/;
+
+These two changes will instruct NGINX that requests sent to `https://example.com/subdir/...` should be forwarded to `http://localhost:8000/...`.
+
+* Inside `server` block, add redirects for well-known URLs (add these lines after `client_max_body_size`, remember to replace `subdir` with your actual subdir!):
+
+		location /.well-known/webfinger { return 301 /subdir$request_uri; }
+		location /.well-known/nodeinfo  { return 301 /subdir$request_uri; }
+		location /.well-known/oauth-authorization-server  { return 301 /subdir$request_uri; }
+
+* Optionally, [check robots.txt from a running microblogpub instance](https://microblog.pub/robots.txt) and integrate it into robots.txt file in the root of your server - remember to prepend `subdir` to URLs, so for example `Disallow: /admin` becomes `Disallow: /subdir/admin`.
+
+## YunoHost edition
+
+[YunoHost](https://yunohost.org/) support is available (although it is not an official package for now): <https://git.sr.ht/~tsileo/microblog.pub_ynh>.
+
+## Available tutorial/guides
+
+ - [Opalstack](https://community.opalstack.com/d/1055-howto-install-and-run-microblogpub-on-opalstack), thanks to [@defulmere@mastodon.social](https://mastodon.online/@defulmere).

docs/templates/layout.html

@@ -63,7 +63,7 @@ nav a:hover, main a:hover, header p a:hover {
     max-width: 960px;
     margin: 50px auto;
 }
-pre code {
+pre {
     padding: 10px;
     overflow: auto;
     display: block;

docs/user_guide.md

@@ -25,33 +25,138 @@ As these two config items define your ActivityPub handle `@handle@domain`.
 
 You can tweak your profile by tweaking these items:
 
- - `name`
- - `summary` (using Markdown)
- - `icon_url`
+ - `name`: The name shown with your profile.
+ - `summary`: The summary or 'bio' part of your profile, written in Markdown.
+ - `icon_url`: Your profile image or avatar.
+ - `image_url`: This provides a 'header' or 'banner' image. Note that it is not shown by the default Microblog.pub templates. It will be used by Mastodon (which uses a 3:1 ratio image) and Pleroma. Pixelfed and Peertube, for example, don't show these images by default.
 
-Whenever one of these config items is updated, an `Update` activity will be sent to all know server to update your remote profile.
+Whenever one of these config items is updated, an `Update` activity will be sent to all known servers to update your remote profile.
+
+The server will need to be restarted for taking changes into account.
+
+Before restarting the server, you can ensure you haven't made any mistakes by running the [configuration checking task](/user_guide.html#configuration-checking).
+
+Note that currently `image_url` is not used anywhere in microblog.pub itself, but other clients/servers do occasionally use it when showing remote profiles as a background image.
+Also, this image _can_ be used in microblog.pub - just add this:
+
+```html
+<img src="{{ local_actor.image_url | media_proxy_url }}">
+```
+
+to an appropriate place of your template (most likely, `header.html`).
+For more information, see a section about [custom templates](/user_guide.html#custom-templates) further in this document.
+
+### Profile metadata
+
+You can add metadata to your profile with the `metadata` config item.
+
+Markdown is supported in the `value` field.
+
+Be aware that most other software like Mastodon will limit the number of key/value to 4.
+
+```toml
+metadata = [
+  {key = "Documentation", value = "[https://docs.microblog.pub](https://docs.microblog.pub)"},
+  {key = "Source code", value = "[https://sr.ht/~tsileo/microblog.pub/](https://sr.ht/~tsileo/microblog.pub/)"},
+]
+```
+
+### Manually approving followers
+
+If you wish to manually approve followers, add this config item to `profile.toml`:
+
+```toml
+manually_approves_followers = true
+```
+
+The default value is `false`.
+
+### Hiding followers
+
+If you wish to hide your followers, add this config item to `profile.toml`:
+
+```toml
+hides_followers = true
+```
+
+The default value is `false`.
+
+### Hiding who you are following
+
+If you wish to hide who you are following, add this config item to `profile.toml`:
+
+```toml
+hides_following = true
+```
+
+The default value is `false`.
 
 ### Privacy replace
 
-You can define domain to be rewrited to more "privacy friendly" alternatives, like [Invidious](https://invidious.io/)
+You can define domains to be rewritten to more "privacy friendly" alternatives, like [Invidious](https://invidious.io/)
 or [Nitter](https://nitter.net/about).
 
-To do so, just add as these extra config items, this is a sample config that rewrite URLs for Twitter, Youtube, Reddit and Medium:
+To do so, add these extra config items. This is a sample config that rewrite URLs for Twitter, Youtube, Reddit and Medium:
 
 ```toml
 privacy_replace = [
     {domain = "youtube.com", replace_by  = "yewtu.be"},
+    {domain = "youtu.be", replace_by  = "yewtu.be"},
     {domain = "twitter.com", replace_by = "nitter.fdn.fr"},
     {domain = "medium.com", replace_by = "scribe.rip"},
     {domain = "reddit.com", replace_by = "teddit.net"},
 ]
 ```
 
+### Disabling certain notification types
+
+All notifications are enabled by default.
+
+You can disabled specific notifications by adding them to the `disabled_notifications` list.
+
+This example disables likes and shares notifications:
+
+```
+disabled_notifications = ["like", "announce"]
+```
+
+#### Available notification types
+
+ - `new_follower`
+ - `rejected_follower`
+ - `unfollow`
+ - `follow_request_accepted`
+ - `follow_request_rejected`
+ - `move`
+ - `like`
+ - `undo_like`
+ - `announce`
+ - `undo_announce`
+ - `mention`
+ - `new_webmention`
+ - `updated_webmention`
+ - `deleted_webmention`
+ - `blocked`
+ - `unblocked`
+ - `block`
+ - `unblock`
+
 ### Customization
 
+#### Default emoji
+
+If you don't like cats, or need more emoji, you can add your favorite emoji in `profile.toml` and it will replace the default ones:
+
+```
+emoji = "🙂🐹📌"
+```
+
+You can copy/paste them from [getemoji.com](https://getemoji.com/).
+
 #### Custom emoji
 
 You can add custom emoji in the `data/custom_emoji` directory and they will be picked automatically.
+Do not use exotic characters in filename - only letters, numbers, and underscore symbol `_` are allowed.
 
 #### Custom CSS
 
@@ -64,19 +169,82 @@ $primary-color: #e14eea;
 $secondary-color: #32cd32;
 ```
 
-See `app/scss/main.scss` to see what variables can be overidden.
+See `app/scss/main.scss` to see what variables can be overridden.
+
+You will need to [recompile CSS](#recompiling-css-files) after doing any CSS changes (for actual css files to be updates) and restart microblog.pub (for css link in HTML documents to be updated with a new checksum - otherwise, browsers that downloaded old CSS will keep using it).
+
+#### Custom favicon
+
+By default, microblog.pub favicon is a square of `$primary-color` CSS color (see above section on how to redefine CSS colors).
+You can change it to any icon you like - just save a desired file as `data/favicon.ico`.
+After that, run the "[recompile CSS](#recompiling-css-files)" task to copy it to `app/static/favicon.ico`.
+
+#### Custom templates
+
+If you'd like to customize your instance's theme beyond CSS, you can modify the app's HTML by placing templates in `data/templates` which overwrite the defaults in `app/templates`.
+
+Templates are written using [Jinja](https://jinja.palletsprojects.com/en/latest/templates/) templating language.
+Moreover, `utils.html` has scoped blocks around the body of every macro.
+This allows macros to be overridden individually in `data/templates/utils.html`, without copying the whole file.
+For example, to only override the display of a specific actor's name/icon, you can create `data/templates/utils.html` file with following content:
+
+```jinja
+{% extends "app/utils.html" %}
+
+{% block display_actor %}
+	{% if actor.ap_id == "https://me.example.com" %}
+		<!-- custom actor display -->
+	{% else %}
+		{{ super() }}
+	{% endif %}
+{% endblock %}
+```
+
+#### Custom Content Security Policy (CSP)
+
+You can override the default Content Security Policy by adding a line in `data/profile.toml`:
+
+```toml
+custom_content_security_policy = "default-src 'self'; style-src 'self' 'sha256-{HIGHLIGHT_CSS_HASH}'; frame-ancestors 'none'; base-uri 'self'; form-action 'self';"
+```
+
+This example will output the default CSP, note that `{HIGHLIGHT_CSS_HASH}` will be dynamically replaced by the correct value (the hash of the CSS needed for syntax highlighting).
+
+#### Code highlighting theme
+
+You can switch to one of the [styles supported by Pygments](https://pygments.org/styles/) by adding a line in `data/profile.toml`:
+
+```toml
+code_highlighting_theme = "solarized-dark"
+```
+
+### Blocking servers
+
+In addition to blocking "single actors" via the admin interface, you can also prevent any communication with entire servers.
+
+Add a `blocked_servers` config item into `profile.toml`.
+
+The `reason` field is just there to help you document/remember why a server was blocked.
+
+You should unfollow any account from a server before blocking it.
+
+```toml
+blocked_servers = [
+    {hostname = "bad.tld", reason = "Bot spam"},
+]
+```
 
 ## Public website
 
 Public notes will be visible on the homepage.
 
-Only the last 20 followers/follows you be showing on the public website.
+Only the last 20 followers/follows you have will be shown on the public website.
 
 And only the last 20 interactions (likes/shares/webmentions) will be displayed, to keep things simple/clean.
 
 ## Admin section
 
-You can login to the admin section by clicking on the `Admin` link in the footer or by visiting `https://yourdomain.tld/admin`.
+You can login to the admin section by clicking on the `Admin` link in the footer or by visiting `https://yourdomain.tld/admin/login`.
 The password is the one set during the initial configuration.
 
 ### Lookup
@@ -129,35 +297,36 @@ microblog.pub supports the most common interactions supported by the Fediverse.
 
 ### Shares
 
-Sharing an object will relay it to your followers and notify the author.
+Sharing (or announcing) an object will relay it to your followers and notify the author.
 It will also be displayed on the homepage.
 
 Most receiving servers will increment the number of shares.
 
-TODO receiving
+Receiving a share will trigger a notification, increment the shares counter on the object and the actor avatar will be displayed on the object permalink.
 
 ### Likes
 
 Liking an object will notify the author.
-Unkike sharing, liked object are not displayed on the homepage.
+
+Unlike sharing, liked objects are not displayed on the homepage.
 
 Most receiving servers will increment the number of likes.
 
-TODO receiving
+Receiving a like will trigger a notification, increment the likes counter on the object and the actor avatar will be displayed on the object permalink.
 
 ### Bookmarks
 
 Bookmarks allow you to like objects without notifying the author.
 
-It is basically a "private like", and allow you to easily access them later.
+It is basically a "private like", and allows you to easily access them later.
 
-TODO receiving
+It will also prevent objects to be pruned.
 
 ### Webmentions
 
-Sending webmention to ping mentioned website is done automatically once a note is authored, see TODO.
+Sending webmentions to ping mentioned websites is done automatically once a public note is authored.
 
-TODO side-effect of receiving a webmention.
+Receiving a webmention will trigger a notification, increment the webmentions counter on the object and the source page will be displayed on the object permalink.
 
 ## Backup and restore
 
@@ -171,3 +340,225 @@ All the data generated by the server is located in the `data/` directory:
  - Uploaded media
 
 Restoring is as easy as adding your backed up `data/` directory into a fresh deployment.
+
+## Moving from another instance
+
+If you want to move followers from your existing account, ensure it is supported in your software documentation.
+
+For [Mastodon you can look at Moving or leaving accounts](https://docs.joinmastodon.org/user/moving/).
+
+If you wish to move **to** another instance, see [Moving to another instance](/user_guide.html#moving-to-another-instance).
+
+First you need to grab the "ActivityPub actor URL" for your existing account:
+
+### Python edition
+
+```bash
+# For a Python install
+poetry run inv webfinger username@instance-you-want-to-move-from.tld
+```
+
+Edit the config.
+
+### Docker edition
+
+```bash
+# For a Docker install
+make account=username@instance-you-want-to-move-from.tld webfinger
+```
+
+Edit the config.
+
+### Edit the config
+
+And add a reference to your old/existing account in `profile.toml`:
+
+```toml
+also_known_as = "https://instance-you-want-to-move-form.tld/users/username"
+```
+
+Restart the server, and you should be able to complete the move from your existing account.
+
+Note that if you already have a redirect in place on Mastodon, you may have to remove it before initiating the migration.
+
+## Import follows from Mastodon
+
+You can import the list of follows/following accounts from Mastodon.
+
+It requires downloading the "Follows" CSV file from your Mastodon instance via "Settings" / "Import and export" / "Data export".
+
+Then you need to run the import task:
+
+### Python edition
+
+```bash
+# For a Python install
+poetry run inv import-mastodon-following-accounts following_accounts.csv
+```
+
+### Docker edition
+
+```bash
+# For a Docker install
+make path=following_accounts.csv import-mastodon-following-accounts
+```
+
+## Tasks
+
+### Configuration checking
+
+You can confirm that your configuration file (`data/profile.toml`) is valid using the `check-config`
+
+#### Python edition
+
+```bash
+poetry run inv check-config
+```
+
+#### Docker edition
+
+```bash
+make check-config
+```
+
+### Recompiling CSS files
+
+You can ensure your custom theme is valid by recompiling the CSS manually using the `compile-scss` task.
+
+#### Python edition
+
+```bash
+poetry run inv compile-scss
+```
+
+#### Docker edition
+
+```bash
+make compile-scss
+```
+
+
+### Password reset
+
+If have lost your password, you can generate a new one using the `reset-password` task.
+
+#### Python edition
+
+```bash
+# shutdown supervisord
+poetry run inv reset-password
+# edit data/profile.toml
+# restart supervisord
+```
+
+#### Docker edition
+
+```bash
+docker compose stop
+make reset-password
+# edit data/profile.toml
+docker compose up -d
+```
+
+### Pruning old data
+
+You should prune old data from time to time to free disk space.
+
+The default retention for the inbox data is 15 days.
+
+It's configurable via the `inbox_retention_days` config item in `profile.toml`:
+
+```toml
+inbox_retention_days = 30
+```
+
+Data owned by the server will never be deleted (at least for now), along with:
+
+ - bookmarked objects
+ - liked objects
+ - shared objects
+ - inbox objects mentioning the local actor
+ - objects related to local conversations (i.e. direct messages, replies) 
+
+For now, it's recommended to make a backup before running the task in case it deletes unwanted data.
+
+You should shutdown the server before running the task.
+
+#### Python edition
+
+```bash
+# shutdown supervisord
+cp -r data/microblogpub.db data/microblogpub.db.bak
+poetry run inv prune-old-data
+# relaunch supervisord and ensure it works as expected
+rm data/microblogpub.db.bak
+```
+
+#### Docker edition
+
+```bash
+docker compose stop
+cp -r data/microblogpub.db data/microblogpub.db.bak
+make prune-old-data
+docker compose up -d
+rm data/microblogpub.db.bak
+```
+
+### Moving to another instance
+
+If you want to migrate to another instance, you have the ability to move your existing followers to your new account.
+
+Your new account should reference the existing one, refer to your software configuration (for example [Moving or leaving accounts from the Mastodon doc](https://docs.joinmastodon.org/user/moving/)).
+
+If you wish to move **from** another instance, see [Moving from another instance](/user_guide.html#moving-from-another-instance).
+
+Execute the Move task:
+
+#### Python edition
+
+```bash
+# For a Python install
+poetry run inv move-to username@domain.tld
+```
+
+#### Docker edition
+
+```bash
+# For a Docker install
+make account=username@domain.tld move-to
+```
+
+### Deleting the instance
+
+If you want to delete your instance, you can request other instances to delete your remote profile.
+
+Note that this is a best-effort delete as some instances may not delete your data.
+
+The command won't remove any local data, it just broadcasts account deletion messages to all known servers.
+
+After executing the command, you should let the server run until all the outgoing delete tasks are sent.
+
+Once deleted, you won't be able to use your instance anymore, but you will be able to perform a fresh re-install of any ActivityPub software.
+
+#### Python edition
+
+```bash
+# For a Python install
+poetry run inv self-destruct
+```
+
+#### Docker edition
+
+```bash
+# For a Docker install
+make self-destruct
+```
+
+## Troubleshooting
+
+If the server is not (re)starting, you can:
+
+ - [Ensure that the configuration is valid](/user_guide.html#configuration-checking).
+ - [Verify if you haven't any syntax error in the custom theme by recompiling the CSS](/user_guide.html#recompiling-css-files).
+ - Look at the log files (in `data/uvicorn.log`, `data/incoming.log` and `data/outgoing.log`).
+ - If the CSS is not working, ensure your reverse proxy is serving the static file correctly.

misc/docker-supervisord.conf

@@ -4,11 +4,10 @@ logfile=/dev/null
 logfile_maxbytes=0
 pidfile=data/supervisord.pid
 
-[fcgi-program:uvicorn]
-socket=tcp://0.0.0.0:8000
-command=uvicorn app.main:app --no-server-header --fd 0
-numprocs=2
-process_name=uvicorn-%(process_num)d
+[program:uvicorn]
+command=uvicorn app.main:app --no-server-header --host 0.0.0.0
+numprocs=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=data/uvicorn.log
 stdout_logfile_maxbytes=50MB
@@ -16,6 +15,7 @@ stdout_logfile_maxbytes=50MB
 [program:incoming_worker]
 command=inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=data/incoming.log
 stdout_logfile_maxbytes=50MB
@@ -23,6 +23,7 @@ stdout_logfile_maxbytes=50MB
 [program:outgoing_worker]
 command=inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=data/outgoing.log
 stdout_logfile_maxbytes=50MB

misc/supervisord.conf

@@ -1,24 +1,25 @@
 [supervisord]
 
-[fcgi-program:uvicorn]
-socket=tcp://localhost:8000
-command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header --fd 0
-numprocs=2
-process_name=uvicorn-%(process_num)d
+[program:uvicorn]
+command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header
+numprocs=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=uvicorn.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB
 
 [program:incoming_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=incoming_worker.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB
 
 [program:outgoing_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
 stdout_logfile=outgoing_worker.log
-stdout_logfile_maxbytes=0
+stdout_logfile_maxbytes=50MB

misc/ynh-supervisord.conf

@@ -1,24 +1,26 @@
 [supervisord]
 
-[fcgi-program:uvicorn]
-socket=tcp://localhost:%(ENV_UVICORN_PORT)s
-command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header --fd 0
-numprocs=2
+[program:uvicorn]
+command=%(ENV_VENV_DIR)s/bin/uvicorn app.main:app --no-server-header
+numprocs=1
+autorestart=true
 process_name=uvicorn-%(process_num)d
 redirect_stderr=true
-stdout_logfile=uvicorn.log
+stdout_logfile=%(ENV_LOG_PATH)s/uvicorn.log
 stdout_logfile_maxbytes=0
 
 [program:incoming_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-incoming-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
-stdout_logfile=incoming_worker.log
+stdout_logfile=%(ENV_LOG_PATH)s/incoming.log
 stdout_logfile_maxbytes=0
 
 [program:outgoing_worker]
 command=%(ENV_VENV_DIR)s/bin/inv process-outgoing-activities
 numproc=1
+autorestart=true
 redirect_stderr=true
-stdout_logfile=outgoing_worker.log
+stdout_logfile=%(ENV_LOG_PATH)s/outgoing.log
 stdout_logfile_maxbytes=0

pyproject.toml

@@ -9,17 +9,15 @@ license = "AGPL-3.0"
 python = "^3.10"
 Jinja2 = "^3.1.2"
 fastapi = "^0.78.0"
-uvicorn = "^0.17.6"
 pycryptodome = "^3.14.1"
 bcrypt = "^3.2.2"
 itsdangerous = "^2.1.2"
 python-multipart = "^0.0.5"
 tomli = "^2.0.1"
-httpx = {extras = ["http2"], version = "^0.23.0"}
+httpx = {version = "0.23.0", extras = ["http2"]}
 SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"}
 alembic = "^1.8.0"
 bleach = "^5.0.0"
-Markdown = "^3.3.7"
 prompt-toolkit = "^3.0.29"
 tomli-w = "^1.0.0"
 python-dateutil = "^2.8.2"
@@ -28,7 +26,6 @@ html5lib = "^1.1"
 mf2py = "^1.1.2"
 Pygments = "^2.12.0"
 loguru = "^0.6.0"
-mdx-linkify = "^2.1"
 Pillow = "^9.1.1"
 blurhash-python = "^1.1.3"
 html2text = "^2020.1.16"
@@ -43,6 +40,11 @@ asgiref = "^3.5.2"
 supervisor = "^4.2.4"
 invoke = "^1.7.1"
 boussole = "^2.0.0"
+uvicorn = {extras = ["standard"], version = "^0.18.3"}
+Brotli = "^1.0.9"
+greenlet = "^1.1.3"
+mistletoe = "^0.9.0"
+Pebble = "^5.0.2"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"

scripts/build_docs.py

@@ -1,19 +1,115 @@
+import re
 import shutil
+import typing
 from pathlib import Path
+from typing import Any
 
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 from jinja2 import select_autoescape
-from markdown import markdown
+from mistletoe import Document  # type: ignore
+from mistletoe import HTMLRenderer  # type: ignore
+from mistletoe import block_token  # type: ignore
+from pygments import highlight  # type: ignore
+from pygments.formatters import HtmlFormatter  # type: ignore
+from pygments.lexers import get_lexer_by_name as get_lexer  # type: ignore
+from pygments.lexers import guess_lexer  # type: ignore
 
 from app.config import VERSION
+from app.source import CustomRenderer
 from app.utils.datetime import now
 
+_FORMATTER = HtmlFormatter()
+_FORMATTER.noclasses = True
 
-def markdownify(content: str) -> str:
-    return markdown(
-        content, extensions=["mdx_linkify", "fenced_code", "codehilite", "toc"]
+
+class DocRenderer(CustomRenderer):
+    def __init__(
+        self,
+        depth=5,
+        omit_title=True,
+        filter_conds=[],
+    ) -> None:
+        super().__init__(
+            enable_mentionify=False,
+            enable_hashtagify=False,
         )
+        self._headings: list[tuple[int, str, str]] = []
+        self._ids: set[str] = set()
+        self.depth = depth
+        self.omit_title = omit_title
+        self.filter_conds = filter_conds
+
+    @property
+    def toc(self):
+        """
+        Returns table of contents as a block_token.List instance.
+        """
+
+        def get_indent(level):
+            if self.omit_title:
+                level -= 1
+            return " " * 4 * (level - 1)
+
+        def build_list_item(heading):
+            level, content, title_id = heading
+            template = '{indent}- <a href="#{id}" rel="nofollow">{content}</a>\n'
+            return template.format(
+                indent=get_indent(level), content=content, id=title_id
+            )
+
+        lines = [build_list_item(heading) for heading in self._headings]
+        items = block_token.tokenize(lines)
+        return items[0]
+
+    def render_heading(self, token):
+        """
+        Overrides super().render_heading; stores rendered heading first,
+        then returns it.
+        """
+        template = '<h{level} id="{id}">{inner}</h{level}>'
+        inner = self.render_inner(token)
+        title_id = inner.lower().replace(" ", "-")
+        if title_id in self._ids:
+            i = 1
+            while 1:
+                title_id = f"{title_id}_{i}"
+                if title_id not in self._ids:
+                    break
+        self._ids.add(title_id)
+        rendered = template.format(level=token.level, inner=inner, id=title_id)
+        content = self.parse_rendered_heading(rendered)
+
+        if not (
+            self.omit_title
+            and token.level == 1
+            or token.level > self.depth
+            or any(cond(content) for cond in self.filter_conds)
+        ):
+            self._headings.append((token.level, content, title_id))
+        return rendered
+
+    @staticmethod
+    def parse_rendered_heading(rendered):
+        """
+        Helper method; converts rendered heading to plain text.
+        """
+        return re.sub(r"<.+?>", "", rendered)
+
+    def render_block_code(self, token: typing.Any) -> str:
+        code = token.children[0].content
+        lexer = get_lexer(token.language) if token.language else guess_lexer(code)
+        return highlight(code, lexer, _FORMATTER)
+
+
+def markdownify(content: str) -> tuple[str, Any]:
+    with DocRenderer() as renderer:
+        rendered_content = renderer.render(Document(content))
+
+    with HTMLRenderer() as html_renderer:
+        toc = html_renderer.render(renderer.toc)
+
+    return rendered_content, toc
 
 
 def main() -> None:
@@ -30,32 +126,36 @@ def main() -> None:
     last_updated = now().replace(second=0, microsecond=0).isoformat()
 
     readme = Path("README.md")
+    content, toc = markdownify(readme.read_text().removeprefix("# microblog.pub"))
     template.stream(
-        content=markdownify(readme.read_text().removeprefix("# microblog.pub")),
+        content=content,
         version=VERSION,
         path="/",
         last_updated=last_updated,
     ).dump("docs/dist/index.html")
 
     install = Path("docs/install.md")
+    content, toc = markdownify(install.read_text())
     template.stream(
-        content=markdownify(install.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/installing.html",
         last_updated=last_updated,
     ).dump("docs/dist/installing.html")
 
     user_guide = Path("docs/user_guide.md")
+    content, toc = markdownify(user_guide.read_text())
     template.stream(
-        content=markdownify(user_guide.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/user_guide.html",
         last_updated=last_updated,
     ).dump("docs/dist/user_guide.html")
 
     developer_guide = Path("docs/developer_guide.md")
+    content, toc = markdownify(developer_guide.read_text())
     template.stream(
-        content=markdownify(developer_guide.read_text()),
+        content=content.replace("[TOC]", toc),
         version=VERSION,
         path="/developer_guide.html",
         last_updated=last_updated,

scripts/config_wizard.py

@@ -6,7 +6,6 @@ from typing import Any
 
 import bcrypt
 import tomli_w
-from markdown import markdown  # type: ignore
 from prompt_toolkit import prompt
 from prompt_toolkit.key_binding import KeyBindings
 
@@ -58,8 +57,7 @@ def main() -> None:
         prompt("admin password: ", is_password=True).encode(), bcrypt.gensalt()
     ).decode()
     dat["name"] = prompt("name (e.g. John Doe): ", default=dat["username"])
-    dat["summary"] = markdown(
-        prompt(
+    dat["summary"] = prompt(
         (
             "summary (short description, in markdown, "
             "press [CTRL] + [SPACE] to submit):\n"
@@ -67,7 +65,6 @@ def main() -> None:
         key_bindings=_kb,
         multiline=True,
     )
-    )
     dat["https"] = True
     proto = "https"
     yn = ""
@@ -78,9 +75,10 @@ def main() -> None:
         proto = "http"
 
     print("Note that you can put your icon/avatar in the static/ directory")
-    dat["icon_url"] = prompt(
+    if icon_url := prompt(
         "icon URL: ", default=f'{proto}://{dat["domain"]}/static/nopic.png'
-    )
+    ):
+        dat["icon_url"] = icon_url
     dat["secret"] = os.urandom(16).hex()
 
     with config_file.open("w") as f:

tasks.py

@@ -1,18 +1,50 @@
 import asyncio
 import io
-import subprocess
+import shutil
 import tarfile
+from collections import namedtuple
 from contextlib import contextmanager
+from inspect import getfullargspec
 from pathlib import Path
 from typing import Generator
 from typing import Optional
+from unittest.mock import patch
 
 import httpx
+import invoke  # type: ignore
 from invoke import Context  # type: ignore
 from invoke import run  # type: ignore
 from invoke import task  # type: ignore
 
 
+def fix_annotations():
+    """
+    Pyinvoke doesn't accept annotations by default, this fix that
+    Based on: @zelo's fix in https://github.com/pyinvoke/invoke/pull/606
+    Context in: https://github.com/pyinvoke/invoke/issues/357
+        Python 3.11 https://github.com/pyinvoke/invoke/issues/833
+    """
+
+    ArgSpec = namedtuple("ArgSpec", ["args", "defaults"])
+
+    def patched_inspect_getargspec(func):
+        spec = getfullargspec(func)
+        return ArgSpec(spec.args, spec.defaults)
+
+    org_task_argspec = invoke.tasks.Task.argspec
+
+    def patched_task_argspec(*args, **kwargs):
+        with patch(
+            target="inspect.getargspec", new=patched_inspect_getargspec, create=True
+        ):
+            return org_task_argspec(*args, **kwargs)
+
+    invoke.tasks.Task.argspec = patched_task_argspec
+
+
+fix_annotations()
+
+
 @task
 def generate_db_migration(ctx, message):
     # type: (Context, str) -> None
@@ -46,7 +78,12 @@ def compile_scss(ctx, watch=False):
     # type: (Context, bool) -> None
     from app.utils.favicon import build_favicon
 
+    favicon_file = Path("data/favicon.ico")
+    if not favicon_file.exists():
         build_favicon()
+    else:
+        shutil.copy2(favicon_file, "app/static/favicon.ico")
+
     theme_file = Path("data/_theme.scss")
     if not theme_file.exists():
         theme_file.write_text("// override vars for theming here")
@@ -164,14 +201,12 @@ def stats(ctx):
 
 @contextmanager
 def embed_version() -> Generator[None, None, None]:
+    from app.utils.version import get_version_commit
+
     version_file = Path("app/_version.py")
     version_file.unlink(missing_ok=True)
-    version = (
-        subprocess.check_output(["git", "rev-parse", "--short=8", "v2"])
-        .split()[0]
-        .decode()
-    )
-    version_file.write_text(f'VERSION_COMMIT = "{version}"')
+    version_commit = get_version_commit()
+    version_file.write_text(f'VERSION_COMMIT = "{version_commit}"')
     try:
         yield
     finally:
@@ -193,6 +228,109 @@ def prune_old_data(ctx):
     asyncio.run(run_prune_old_data())
 
 
+@task
+def webfinger(ctx, account):
+    # type: (Context, str) -> None
+    import traceback
+
+    from loguru import logger
+
+    from app.source import _MENTION_REGEX
+    from app.webfinger import get_actor_url
+
+    logger.disable("app")
+    if not account.startswith("@"):
+        account = f"@{account}"
+    if not _MENTION_REGEX.match(account):
+        print(f"Invalid acccount {account}")
+        return
+
+    print(f"Resolving {account}")
+    try:
+        maybe_actor_url = asyncio.run(get_actor_url(account))
+        if maybe_actor_url:
+            print(f"SUCCESS: {maybe_actor_url}")
+        else:
+            print(f"ERROR: Failed to resolve {account}")
+    except Exception as exc:
+        print(f"ERROR: Failed to resolve {account}")
+        print("".join(traceback.format_exception(exc)))
+
+
+@task
+def move_to(ctx, moved_to):
+    # type: (Context, str) -> None
+    import traceback
+
+    from loguru import logger
+
+    from app.actor import LOCAL_ACTOR
+    from app.actor import fetch_actor
+    from app.boxes import send_move
+    from app.database import async_session
+    from app.source import _MENTION_REGEX
+    from app.webfinger import get_actor_url
+
+    logger.disable("app")
+
+    if not moved_to.startswith("@"):
+        moved_to = f"@{moved_to}"
+    if not _MENTION_REGEX.match(moved_to):
+        print(f"Invalid acccount {moved_to}")
+        return
+
+    async def _send_move():
+        print(f"Initiating move to {moved_to}")
+        async with async_session() as db_session:
+            try:
+                moved_to_actor_id = await get_actor_url(moved_to)
+            except Exception as exc:
+                print(f"ERROR: Failed to resolve {moved_to}")
+                print("".join(traceback.format_exception(exc)))
+                return
+
+            if not moved_to_actor_id:
+                print("ERROR: Failed to resolve {moved_to}")
+                return
+
+            new_actor = await fetch_actor(db_session, moved_to_actor_id)
+
+            if LOCAL_ACTOR.ap_id not in new_actor.ap_actor.get("alsoKnownAs", []):
+                print(
+                    f"{new_actor.handle}/{moved_to_actor_id} is missing "
+                    f"{LOCAL_ACTOR.ap_id} in alsoKnownAs"
+                )
+                return
+
+            await send_move(db_session, new_actor.ap_id)
+
+        print("Done")
+
+    asyncio.run(_send_move())
+
+
+@task
+def self_destruct(ctx):
+    # type: (Context) -> None
+    from loguru import logger
+
+    from app.boxes import send_self_destruct
+    from app.database import async_session
+
+    logger.disable("app")
+
+    async def _send_self_destruct():
+        if input("Initiating self destruct, type yes to confirm: ") != "yes":
+            print("Aborting")
+
+        async with async_session() as db_session:
+            await send_self_destruct(db_session)
+
+        print("Done")
+
+    asyncio.run(_send_self_destruct())
+
+
 @task
 def yunohost_config(
     ctx,
@@ -212,3 +350,75 @@ def yunohost_config(
         summary=summary,
         password=password,
     )
+
+
+@task
+def reset_password(ctx):
+    # type: (Context) -> None
+    import bcrypt
+    from prompt_toolkit import prompt
+
+    new_password = bcrypt.hashpw(
+        prompt("New admin password: ", is_password=True).encode(), bcrypt.gensalt()
+    ).decode()
+
+    print()
+    print("Update data/profile.toml with:")
+    print(f'admin_password = "{new_password}"')
+
+
+@task
+def check_config(ctx):
+    # type: (Context) -> None
+    import sys
+    import traceback
+
+    from loguru import logger
+
+    logger.disable("app")
+
+    try:
+        from app import config  # noqa: F401
+    except Exception as exc:
+        print("Config error, please fix data/profile.toml:\n")
+        print("".join(traceback.format_exception(exc)))
+        sys.exit(1)
+    else:
+        print("Config is OK")
+
+
+@task
+def import_mastodon_following_accounts(ctx, path):
+    # type: (Context, str) -> None
+    from loguru import logger
+
+    from app.boxes import _get_following
+    from app.boxes import _send_follow
+    from app.database import async_session
+    from app.utils.mastodon import get_actor_urls_from_following_accounts_csv_file
+
+    async def _import_following() -> int:
+        count = 0
+        async with async_session() as db_session:
+            followings = {
+                following.ap_actor_id for following in await _get_following(db_session)
+            }
+            for (
+                handle,
+                actor_url,
+            ) in await get_actor_urls_from_following_accounts_csv_file(path):
+                if actor_url in followings:
+                    logger.info(f"Already following {handle}")
+                    continue
+
+                logger.info(f"Importing {actor_url=}")
+
+                await _send_follow(db_session, actor_url)
+                count += 1
+
+            await db_session.commit()
+
+        return count
+
+    count = asyncio.run(_import_following())
+    logger.info(f"Import done, {count} follow requests sent")

tests/factories.py

@@ -68,6 +68,20 @@ def build_accept_activity(
     }
 
 
+def build_block_activity(
+    from_remote_actor: actor.RemoteActor,
+    for_remote_actor: actor.RemoteActor,
+    outbox_public_id: str | None = None,
+) -> ap.RawObject:
+    return {
+        "@context": ap.AS_CTX,
+        "type": "Block",
+        "id": from_remote_actor.ap_id + "/block/" + (outbox_public_id or uuid4().hex),
+        "actor": from_remote_actor.ap_id,
+        "object": for_remote_actor.ap_id,
+    }
+
+
 def build_move_activity(
     from_remote_actor: actor.RemoteActor,
     for_remote_object: actor.RemoteActor,
@@ -84,12 +98,13 @@ def build_move_activity(
 
 
 def build_note_object(
-    from_remote_actor: actor.RemoteActor,
+    from_remote_actor: actor.RemoteActor | models.Actor,
     outbox_public_id: str | None = None,
     content: str = "Hello",
     to: list[str] = None,
     cc: list[str] = None,
     tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
 ) -> ap.RawObject:
     published = now().replace(microsecond=0).isoformat().replace("+00:00", "Z")
     context = from_remote_actor.ap_id + "/ctx/" + uuid4().hex
@@ -108,8 +123,8 @@ def build_note_object(
         "url": from_remote_actor.ap_id + "/note/" + note_id,
         "tag": tags or [],
         "summary": None,
-        "inReplyTo": None,
         "sensitive": False,
+        "inReplyTo": in_reply_to,
     }
 
 

tests/test_actor.py

@@ -20,12 +20,16 @@ async def test_fetch_actor(async_db_session: AsyncSession, respx_mock) -> None:
         public_key="pk",
     )
     respx_mock.get(ra.ap_id).mock(return_value=httpx.Response(200, json=ra.ap_actor))
+    respx_mock.get(
+        "https://example.com/.well-known/webfinger",
+        params={"resource": "acct%3Atoto%40example.com"},
+    ).mock(return_value=httpx.Response(200, json={"subject": "acct:toto@example.com"}))
 
     # When fetching this actor for the first time
     saved_actor = await fetch_actor(async_db_session, ra.ap_id)
 
     # Then it has been fetched and saved in DB
-    assert respx.calls.call_count == 1
+    assert respx.calls.call_count == 2
     assert (
         await async_db_session.execute(select(models.Actor))
     ).scalar_one().ap_id == saved_actor.ap_id
@@ -38,7 +42,7 @@ async def test_fetch_actor(async_db_session: AsyncSession, respx_mock) -> None:
     assert (
         await async_db_session.execute(select(func.count(models.Actor.id)))
     ).scalar_one() == 1
-    assert respx.calls.call_count == 1
+    assert respx.calls.call_count == 2
 
 
 def test_sqlalchemy_factory(db: Session) -> None:
@@ -52,4 +56,4 @@ def test_sqlalchemy_factory(db: Session) -> None:
         ap_actor=ra.ap_actor,
         ap_id=ra.ap_id,
     )
-    assert actor_in_db.id == db.query(models.Actor).one().id
+    assert actor_in_db.id == db.execute(select(models.Actor)).scalar_one().id

tests/test_inbox.py

@@ -75,7 +75,7 @@ def test_inbox_incoming_follow_request(
     assert inbox_object.ap_object == follow_activity.ap_object
 
     # And a follower was internally created
-    follower = db.query(models.Follower).one()
+    follower = db.execute(select(models.Follower)).scalar_one()
     assert follower.ap_actor_id == ra.ap_id
     assert follower.actor_id == saved_actor.id
     assert follower.inbox_object_id == inbox_object.id
@@ -414,3 +414,62 @@ def test_inbox__move_activity(
         )
         == 1
     )
+
+    # And a notification was created
+    notif = db.execute(
+        select(models.Notification).where(
+            models.Notification.notification_type == models.NotificationType.MOVE
+        )
+    ).scalar_one()
+    assert notif.actor.ap_id == new_ra.ap_id
+    assert notif.inbox_object_id == inbox_activity.id
+
+
+def test_inbox__block_activity(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # Given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    # Which is followed by the local actor
+    setup_remote_actor_as_following(ra)
+
+    # When receiving a Block activity
+    follow_activity = RemoteObject(
+        factories.build_block_activity(
+            from_remote_actor=ra,
+            for_remote_actor=LOCAL_ACTOR,
+        ),
+        ra,
+    )
+    with mock_httpsig_checker(ra):
+        response = client.post(
+            "/inbox",
+            headers={"Content-Type": ap.AS_CTX},
+            json=follow_activity.ap_object,
+        )
+
+    # Then the server returns a 202
+    assert response.status_code == 202
+
+    run_process_next_incoming_activity()
+
+    # And the actor was saved in DB
+    saved_actor = db.execute(select(models.Actor)).scalar_one()
+    assert saved_actor.ap_id == ra.ap_id
+
+    # And the Block activity was saved in the inbox
+    inbox_activity = db.execute(
+        select(models.InboxObject).where(models.InboxObject.ap_type == "Block")
+    ).scalar_one()
+
+    # And a notification was created
+    notif = db.execute(
+        select(models.Notification).where(
+            models.Notification.notification_type == models.NotificationType.BLOCKED
+        )
+    ).scalar_one()
+    assert notif.actor.ap_id == ra.ap_id
+    assert notif.inbox_object_id == inbox_activity.id

tests/test_outbox.py

@@ -2,13 +2,17 @@ from unittest import mock
 
 import respx
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app import activitypub as ap
 from app import models
 from app import webfinger
+from app.actor import LOCAL_ACTOR
 from app.config import generate_csrf_token
 from tests.utils import generate_admin_session_cookies
+from tests.utils import setup_inbox_note
+from tests.utils import setup_outbox_note
 from tests.utils import setup_remote_actor
 from tests.utils import setup_remote_actor_as_follower
 
@@ -49,16 +53,184 @@ def test_send_follow_request(
     assert response.headers.get("Location") == "http://testserver/"
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Follow"
     assert outbox_object.activity_object_ap_id == ra.ap_id
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == ra.inbox_url
 
 
+def test_send_delete__reverts_side_effects(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    # who is a follower
+    follower = setup_remote_actor_as_follower(ra)
+    actor = follower.actor
+
+    # with a note that has existing replies
+    inbox_note = setup_inbox_note(actor)
+    # with a bogus counter
+    inbox_note.replies_count = 5
+    db.commit()
+
+    # and 2 local replies
+    setup_outbox_note(
+        to=[ap.AS_PUBLIC],
+        cc=[LOCAL_ACTOR.followers_collection_id],  # type: ignore
+        in_reply_to=inbox_note.ap_id,
+    )
+    outbox_note2 = setup_outbox_note(
+        to=[ap.AS_PUBLIC],
+        cc=[LOCAL_ACTOR.followers_collection_id],  # type: ignore
+        in_reply_to=inbox_note.ap_id,
+    )
+    db.commit()
+
+    # When deleting one of the replies
+    response = client.post(
+        "/admin/actions/delete",
+        data={
+            "redirect_url": "http://testserver/",
+            "ap_object_id": outbox_note2.ap_id,
+            "csrf_token": generate_csrf_token(),
+        },
+        cookies=generate_admin_session_cookies(),
+    )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+    assert response.headers.get("Location") == "http://testserver/"
+
+    # And the Delete activity was created in the outbox
+    outbox_object = db.execute(
+        select(models.OutboxObject).where(models.OutboxObject.ap_type == "Delete")
+    ).scalar_one()
+    assert outbox_object.ap_type == "Delete"
+    assert outbox_object.activity_object_ap_id == outbox_note2.ap_id
+
+    # And an outgoing activity was queued
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
+    assert outgoing_activity.outbox_object_id == outbox_object.id
+    assert outgoing_activity.recipient == ra.inbox_url
+
+    # And the replies count of the replied object was refreshed correctly
+    db.refresh(inbox_note)
+    assert inbox_note.replies_count == 1
+
+
+def test_send_create_activity__no_content(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 422
+    assert response.status_code == 422
+
+
+def test_send_create_activity__with_attachment(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "content": "hello",
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            files=[
+                ("files", ("attachment.txt", "hello")),
+            ],
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+
+    # And the Follow activity was created in the outbox
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
+    assert outbox_object.ap_type == "Note"
+    assert outbox_object.summary is None
+    assert outbox_object.content == "<p>hello</p>\n"
+    assert len(outbox_object.attachments) == 1
+    attachment = outbox_object.attachments[0]
+    assert attachment.type == "Document"
+
+    attachment_response = client.get(attachment.url)
+    assert attachment_response.status_code == 200
+    assert attachment_response.content == b"hello"
+
+    upload = db.execute(select(models.Upload)).scalar_one()
+    assert upload.content_hash == (
+        "324dcf027dd4a30a932c441f365a25e86b173defa4b8e58948253471b81b72cf"
+    )
+
+    outbox_attachment = db.execute(select(models.OutboxObjectAttachment)).scalar_one()
+    assert outbox_attachment.upload_id == upload.id
+    assert outbox_attachment.outbox_object_id == outbox_object.id
+    assert outbox_attachment.filename == "attachment.txt"
+
+
+def test_send_create_activity__no_content_with_cw_and_attachments(
+    db: Session,
+    client: TestClient,
+    respx_mock: respx.MockRouter,
+) -> None:
+    # given a remote actor
+    ra = setup_remote_actor(respx_mock)
+
+    with mock.patch.object(webfinger, "get_actor_url", return_value=ra.ap_id):
+        response = client.post(
+            "/admin/actions/new",
+            data={
+                "content_warning": "cw",
+                "redirect_url": "http://testserver/",
+                "visibility": ap.VisibilityEnum.PUBLIC.name,
+                "csrf_token": generate_csrf_token(),
+            },
+            files={"files": ("attachment.txt", "hello")},
+            cookies=generate_admin_session_cookies(),
+        )
+
+    # Then the server returns a 302
+    assert response.status_code == 302
+
+    # And the Follow activity was created in the outbox
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
+    assert outbox_object.ap_type == "Note"
+    assert outbox_object.summary is None
+    assert outbox_object.content == "<p>cw</p>\n"
+    assert len(outbox_object.attachments) == 1
+
+
 def test_send_create_activity__no_followers_and_with_mention(
     db: Session,
     client: TestClient,
@@ -83,11 +255,11 @@ def test_send_create_activity__no_followers_and_with_mention(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == ra.inbox_url
 
@@ -119,11 +291,11 @@ def test_send_create_activity__with_followers(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -159,7 +331,7 @@ def test_send_create_activity__question__one_of(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Question"
     assert outbox_object.is_one_of_poll is True
     assert len(outbox_object.poll_items) == 2
@@ -167,7 +339,7 @@ def test_send_create_activity__question__one_of(
     assert outbox_object.is_poll_ended is False
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -205,7 +377,7 @@ def test_send_create_activity__question__any_of(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Question"
     assert outbox_object.is_one_of_poll is False
     assert len(outbox_object.poll_items) == 4
@@ -213,7 +385,7 @@ def test_send_create_activity__question__any_of(
     assert outbox_object.is_poll_ended is False
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url
 
@@ -246,11 +418,11 @@ def test_send_create_activity__article(
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Article"
     assert outbox_object.ap_object["name"] == "Article"
 
     # And an outgoing activity was queued
-    outgoing_activity = db.query(models.OutgoingActivity).one()
+    outgoing_activity = db.execute(select(models.OutgoingActivity)).scalar_one()
     assert outgoing_activity.outbox_object_id == outbox_object.id
     assert outgoing_activity.recipient == follower.actor.inbox_url

tests/test_public.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy.orm import Session
@@ -31,7 +33,19 @@ def test_followers__ap(client, db) -> None:
     response = client.get("/followers", headers={"Accept": ap.AP_CONTENT_TYPE})
     assert response.status_code == 200
     assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
-    assert response.json()["id"].endswith("/followers")
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/followers")
+    assert "first" in json_resp
+
+
+def test_followers__ap_hides_followers(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWERS", True):
+        response = client.get("/followers", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 200
+    assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/followers")
+    assert "first" not in json_resp
 
 
 def test_followers__html(client, db) -> None:
@@ -40,14 +54,40 @@ def test_followers__html(client, db) -> None:
     assert response.headers["content-type"].startswith("text/html")
 
 
+def test_followers__html_hides_followers(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWERS", True):
+        response = client.get("/followers", headers={"Accept": "text/html"})
+    assert response.status_code == 404
+    assert response.headers["content-type"].startswith("text/html")
+
+
 def test_following__ap(client, db) -> None:
     response = client.get("/following", headers={"Accept": ap.AP_CONTENT_TYPE})
     assert response.status_code == 200
     assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
-    assert response.json()["id"].endswith("/following")
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/following")
+    assert "first" in json_resp
+
+
+def test_following__ap_hides_following(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWING", True):
+        response = client.get("/following", headers={"Accept": ap.AP_CONTENT_TYPE})
+    assert response.status_code == 200
+    assert response.headers["content-type"] == ap.AP_CONTENT_TYPE
+    json_resp = response.json()
+    assert json_resp["id"].endswith("/following")
+    assert "first" not in json_resp
 
 
 def test_following__html(client, db) -> None:
     response = client.get("/following")
     assert response.status_code == 200
     assert response.headers["content-type"].startswith("text/html")
+
+
+def test_following__html_hides_following(client, db) -> None:
+    with mock.patch("app.main.config.HIDES_FOLLOWING", True):
+        response = client.get("/following", headers={"Accept": "text/html"})
+    assert response.status_code == 404
+    assert response.headers["content-type"].startswith("text/html")

tests/test_tags.py

@@ -1,4 +1,5 @@
 from fastapi.testclient import TestClient
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app import activitypub as ap
@@ -35,7 +36,7 @@ def test_tags__note_with_tag(db: Session, client: TestClient) -> None:
     assert response.status_code == 302
 
     # And the Follow activity was created in the outbox
-    outbox_object = db.query(models.OutboxObject).one()
+    outbox_object = db.execute(select(models.OutboxObject)).scalar_one()
     assert outbox_object.ap_type == "Note"
     assert len(outbox_object.tags) == 1
     emoji_tag = outbox_object.tags[0]

tests/test_utils.py

@@ -0,0 +1,19 @@
+from unittest import mock
+
+import pytest
+
+from app.utils.url import is_hostname_blocked
+
+
+@pytest.mark.parametrize(
+    "hostname,should_be_blocked",
+    [
+        ("example.com", True),
+        ("subdomain.example.com", True),
+        ("example.xyz", False),
+    ],
+)
+def test_is_hostname_blocked(hostname: str, should_be_blocked: bool) -> None:
+    with mock.patch("app.utils.url.BLOCKED_SERVERS", ["example.com"]):
+        is_hostname_blocked.cache_clear()
+        assert is_hostname_blocked(hostname) is should_be_blocked

tests/utils.py

@@ -169,6 +169,53 @@ def setup_remote_actor_as_following_and_follower(
     return following, follower
 
 
+def setup_outbox_note(
+    content: str = "Hello",
+    to: list[str] = None,
+    cc: list[str] = None,
+    tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
+) -> models.OutboxObject:
+    note_id = uuid4().hex
+    note_from_outbox = RemoteObject(
+        factories.build_note_object(
+            from_remote_actor=LOCAL_ACTOR,
+            outbox_public_id=note_id,
+            content=content,
+            to=to,
+            cc=cc,
+            tags=tags,
+            in_reply_to=in_reply_to,
+        ),
+        LOCAL_ACTOR,
+    )
+    return factories.OutboxObjectFactory.from_remote_object(note_id, note_from_outbox)
+
+
+def setup_inbox_note(
+    actor: models.Actor,
+    content: str = "Hello",
+    to: list[str] = None,
+    cc: list[str] = None,
+    tags: list[ap.RawObject] = None,
+    in_reply_to: str | None = None,
+) -> models.OutboxObject:
+    note_id = uuid4().hex
+    note_from_outbox = RemoteObject(
+        factories.build_note_object(
+            from_remote_actor=actor,
+            outbox_public_id=note_id,
+            content=content,
+            to=to,
+            cc=cc,
+            tags=tags,
+            in_reply_to=in_reply_to,
+        ),
+        actor,
+    )
+    return factories.InboxObjectFactory.from_remote_object(note_from_outbox, actor)
+
+
 def setup_inbox_delete(
     actor: models.Actor, deleted_object_ap_id: str
 ) -> models.InboxObject: