From 4bf54c7040cba0da9eb975b585add5304b53837e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 26 Jun 2022 18:07:55 +0200 Subject: [PATCH] Improved audience support and implement featured collection --- ...n.py => ba131b14c3a1_initial_migration.py} | 9 ++- app/activitypub.py | 20 ++++- app/actor.py | 8 ++ app/admin.py | 77 +++++++++++++++++-- app/ap_object.py | 6 +- app/boxes.py | 30 +++++++- app/main.py | 67 +++++++++++++--- app/models.py | 14 ++++ app/scss/main.scss | 3 + app/templates.py | 15 ++-- app/templates/admin_new.html | 9 ++- app/templates/admin_outbox.html | 25 +++++- app/templates/header.html | 1 + app/templates/index.html | 6 ++ app/templates/layout.html | 1 - app/templates/utils.html | 30 +++++++- 16 files changed, 284 insertions(+), 37 deletions(-) rename alembic/versions/{714b4a5307c7_initial_migration.py => ba131b14c3a1_initial_migration.py} (97%) diff --git a/alembic/versions/714b4a5307c7_initial_migration.py b/alembic/versions/ba131b14c3a1_initial_migration.py similarity index 97% rename from alembic/versions/714b4a5307c7_initial_migration.py rename to alembic/versions/ba131b14c3a1_initial_migration.py index ff6f866..627a89d 100644 --- a/alembic/versions/714b4a5307c7_initial_migration.py +++ b/alembic/versions/ba131b14c3a1_initial_migration.py @@ -1,8 +1,8 @@ """Initial migration -Revision ID: 714b4a5307c7 +Revision ID: ba131b14c3a1 Revises: -Create Date: 2022-06-23 18:42:56.009810 +Create Date: 2022-06-26 14:36:44.107422 """ import sqlalchemy as sa @@ -10,7 +10,7 @@ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. -revision = '714b4a5307c7' +revision = 'ba131b14c3a1' down_revision = None branch_labels = None depends_on = None @@ -81,10 +81,13 @@ def upgrade() -> None: sa.Column('replies_count', sa.Integer(), nullable=False), sa.Column('webmentions', sa.JSON(), nullable=True), sa.Column('og_meta', sa.JSON(), nullable=True), + sa.Column('is_pinned', sa.Boolean(), nullable=False), sa.Column('is_deleted', sa.Boolean(), nullable=False), sa.Column('relates_to_inbox_object_id', sa.Integer(), nullable=True), sa.Column('relates_to_outbox_object_id', sa.Integer(), nullable=True), + sa.Column('relates_to_actor_id', sa.Integer(), nullable=True), sa.Column('undone_by_outbox_object_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['relates_to_actor_id'], ['actor.id'], ), sa.ForeignKeyConstraint(['relates_to_inbox_object_id'], ['inbox.id'], ), sa.ForeignKeyConstraint(['relates_to_outbox_object_id'], ['outbox.id'], ), sa.ForeignKeyConstraint(['undone_by_outbox_object_id'], ['outbox.id'], ), diff --git a/app/activitypub.py b/app/activitypub.py index 16dee4e..7cf0d9f 100644 --- a/app/activitypub.py +++ b/app/activitypub.py @@ -1,6 +1,7 @@ import enum import json import mimetypes +from typing import TYPE_CHECKING from typing import Any import httpx @@ -10,6 +11,9 @@ from app.config import AP_CONTENT_TYPE # noqa: F401 from app.httpsig import auth from app.key import get_pubkey_as_pem +if TYPE_CHECKING: + from app.actor import Actor + RawObject = dict[str, Any] AS_CTX = "https://www.w3.org/ns/activitystreams" AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" @@ -24,8 +28,18 @@ class ObjectIsGoneError(Exception): class VisibilityEnum(str, enum.Enum): PUBLIC = "public" UNLISTED = "unlisted" + FOLLOWERS_ONLY = "followers-only" DIRECT = "direct" + @staticmethod + def get_display_name(key: "VisibilityEnum") -> str: + return { + VisibilityEnum.PUBLIC: "Public - sent to followers and visible on the homepage", # noqa: E501 + VisibilityEnum.UNLISTED: "Unlisted - like public, but hidden from the homepage", # noqa: E501, + VisibilityEnum.FOLLOWERS_ONLY: "Followers only", + VisibilityEnum.DIRECT: "Direct - only visible for mentioned actors", + }[key] + MICROBLOGPUB = { "@context": [ @@ -70,7 +84,7 @@ ME = { "id": config.ID, "following": config.BASE_URL + "/following", "followers": config.BASE_URL + "/followers", - # "featured": ID + "/featured", + "featured": config.BASE_URL + "/featured", "inbox": config.BASE_URL + "/inbox", "outbox": config.BASE_URL + "/outbox", "preferredUsername": config.USERNAME, @@ -198,13 +212,15 @@ def get_id(val: str | dict[str, Any]) -> str: return val -def object_visibility(ap_activity: RawObject) -> VisibilityEnum: +def object_visibility(ap_activity: RawObject, actor: "Actor") -> VisibilityEnum: to = as_list(ap_activity.get("to", [])) cc = as_list(ap_activity.get("cc", [])) if AS_PUBLIC in to: return VisibilityEnum.PUBLIC elif AS_PUBLIC in cc: return VisibilityEnum.UNLISTED + elif actor.followers_collection_id in to + cc: + return VisibilityEnum.FOLLOWERS_ONLY else: return VisibilityEnum.DIRECT diff --git a/app/actor.py b/app/actor.py index 805d466..134f7e1 100644 --- a/app/actor.py +++ b/app/actor.py @@ -97,6 +97,14 @@ class Actor: else: return "/static/nopic.png" + @property + def tags(self) -> list[ap.RawObject]: + return self.ap_actor.get("tag", []) + + @property + def followers_collection_id(self) -> str: + return self.ap_actor["followers"] + class RemoteActor(Actor): def __init__(self, ap_actor: ap.RawObject) -> None: diff --git a/app/admin.py b/app/admin.py index 998f0c3..01f3bd9 100644 --- a/app/admin.py +++ b/app/admin.py @@ -13,8 +13,10 @@ from app import activitypub as ap from app import boxes from app import models from app import templates +from app.actor import LOCAL_ACTOR from app.actor import get_actors_metadata from app.boxes import get_inbox_object_by_ap_id +from app.boxes import get_outbox_object_by_ap_id from app.boxes import send_follow from app.config import generate_csrf_token from app.config import session_serializer @@ -96,17 +98,32 @@ def admin_new( in_reply_to: str | None = None, db: Session = Depends(get_db), ) -> templates.TemplateResponse: + content = "" in_reply_to_object = None if in_reply_to: in_reply_to_object = boxes.get_anybox_object_by_ap_id(db, in_reply_to) + + # Add mentions to the initial note content if not in_reply_to_object: raise ValueError(f"Unknown object {in_reply_to=}") + if in_reply_to_object.actor.ap_id != LOCAL_ACTOR.ap_id: + content += f"{in_reply_to_object.actor.handle} " + for tag in in_reply_to_object.tags: + if tag.get("type") == "Mention" and tag["name"] != LOCAL_ACTOR.handle: + content += f'{tag["name"]} ' return templates.render_template( db, request, "admin_new.html", - {"in_reply_to_object": in_reply_to_object}, + { + "in_reply_to_object": in_reply_to_object, + "content": content, + "visibility_enum": [ + (v.name, ap.VisibilityEnum.get_display_name(v)) + for v in ap.VisibilityEnum + ], + }, ) @@ -194,24 +211,39 @@ def admin_inbox( @router.get("/outbox") def admin_outbox( - request: Request, - db: Session = Depends(get_db), + request: Request, db: Session = Depends(get_db), filter_by: str | None = None ) -> templates.TemplateResponse: + q = db.query(models.OutboxObject).filter( + models.OutboxObject.ap_type.not_in(["Accept"]) + ) + if filter_by: + q = q.filter(models.OutboxObject.ap_type == filter_by) + outbox = ( - db.query(models.OutboxObject) - .options( + q.options( joinedload(models.OutboxObject.relates_to_inbox_object), joinedload(models.OutboxObject.relates_to_outbox_object), + joinedload(models.OutboxObject.relates_to_actor), ) .order_by(models.OutboxObject.ap_published_at.desc()) .limit(20) .all() ) + actors_metadata = get_actors_metadata( + db, + [ + outbox_object.relates_to_actor + for outbox_object in outbox + if outbox_object.relates_to_actor + ], + ) + return templates.render_template( db, request, "admin_outbox.html", { + "actors_metadata": actors_metadata, "outbox": outbox, }, ) @@ -288,6 +320,7 @@ def admin_profile( models.InboxObject.actor_id == actor.id, models.InboxObject.ap_type.in_(["Note", "Article", "Video"]), ) + .order_by(models.InboxObject.ap_published_at.desc()) .all() ) @@ -384,6 +417,38 @@ def admin_actions_unbookmark( return RedirectResponse(redirect_url, status_code=302) +@router.post("/actions/pin") +def admin_actions_pin( + request: Request, + ap_object_id: str = Form(), + redirect_url: str = Form(), + csrf_check: None = Depends(verify_csrf_token), + db: Session = Depends(get_db), +) -> RedirectResponse: + outbox_object = get_outbox_object_by_ap_id(db, ap_object_id) + if not outbox_object: + raise ValueError("Should never happen") + outbox_object.is_pinned = True + db.commit() + return RedirectResponse(redirect_url, status_code=302) + + +@router.post("/actions/unpin") +def admin_actions_unpin( + request: Request, + ap_object_id: str = Form(), + redirect_url: str = Form(), + csrf_check: None = Depends(verify_csrf_token), + db: Session = Depends(get_db), +) -> RedirectResponse: + outbox_object = get_outbox_object_by_ap_id(db, ap_object_id) + if not outbox_object: + raise ValueError("Should never happen") + outbox_object.is_pinned = False + db.commit() + return RedirectResponse(redirect_url, status_code=302) + + @router.post("/actions/new") def admin_actions_new( request: Request, @@ -391,6 +456,7 @@ def admin_actions_new( content: str = Form(), redirect_url: str = Form(), in_reply_to: str | None = Form(None), + visibility: str = Form(), csrf_check: None = Depends(verify_csrf_token), db: Session = Depends(get_db), ) -> RedirectResponse: @@ -405,6 +471,7 @@ def admin_actions_new( source=content, uploads=uploads, in_reply_to=in_reply_to or None, + visibility=ap.VisibilityEnum[visibility], ) return RedirectResponse( request.url_for("outbox_by_public_id", public_id=public_id), diff --git a/app/ap_object.py b/app/ap_object.py index 69796f2..59d0187 100644 --- a/app/ap_object.py +++ b/app/ap_object.py @@ -58,7 +58,7 @@ class Object: @property def visibility(self) -> ap.VisibilityEnum: - return ap.object_visibility(self.ap_object) + return ap.object_visibility(self.ap_object, self.actor) @property def ap_context(self) -> str | None: @@ -68,6 +68,10 @@ class Object: def sensitive(self) -> bool: return self.ap_object.get("sensitive", False) + @property + def tags(self) -> list[ap.RawObject]: + return self.ap_object.get("tag", []) + @property def attachments(self) -> list["Attachment"]: attachments = [] diff --git a/app/boxes.py b/app/boxes.py index 6c26385..aab782b 100644 --- a/app/boxes.py +++ b/app/boxes.py @@ -43,6 +43,7 @@ def save_outbox_object( raw_object: ap.RawObject, relates_to_inbox_object_id: int | None = None, relates_to_outbox_object_id: int | None = None, + relates_to_actor_id: int | None = None, source: str | None = None, ) -> models.OutboxObject: ra = RemoteObject(raw_object) @@ -57,6 +58,7 @@ def save_outbox_object( og_meta=ra.og_meta, relates_to_inbox_object_id=relates_to_inbox_object_id, relates_to_outbox_object_id=relates_to_outbox_object_id, + relates_to_actor_id=relates_to_actor_id, activity_object_ap_id=ra.activity_object_ap_id, is_hidden_from_homepage=True if ra.in_reply_to else False, ) @@ -136,7 +138,9 @@ def send_follow(db: Session, ap_actor_id: str) -> None: "object": ap_actor_id, } - outbox_object = save_outbox_object(db, follow_id, follow) + outbox_object = save_outbox_object( + db, follow_id, follow, relates_to_actor_id=actor.id + ) if not outbox_object.id: raise ValueError("Should never happen") @@ -224,6 +228,7 @@ def send_create( source: str, uploads: list[tuple[models.Upload, str]], in_reply_to: str | None, + visibility: ap.VisibilityEnum, ) -> str: note_id = allocate_outbox_id() published = now().replace(microsecond=0).isoformat().replace("+00:00", "Z") @@ -247,14 +252,33 @@ def send_create( for (upload, filename) in uploads: attachments.append(upload_to_attachment(upload, filename)) + mentioned_actors = [ + mention["href"] for mention in tags if mention["type"] == "Mention" + ] + + to = [] + cc = [] + if visibility == ap.VisibilityEnum.PUBLIC: + to = [ap.AS_PUBLIC] + cc = [f"{BASE_URL}/followers"] + mentioned_actors + elif visibility == ap.VisibilityEnum.UNLISTED: + to = [f"{BASE_URL}/followers"] + cc = [ap.AS_PUBLIC] + mentioned_actors + elif visibility == ap.VisibilityEnum.FOLLOWERS_ONLY: + to = [f"{BASE_URL}/followers"] + cc = mentioned_actors + elif visibility == ap.VisibilityEnum.DIRECT: + to = mentioned_actors + cc = [] + note = { "@context": ap.AS_CTX, "type": "Note", "id": outbox_object_id(note_id), "attributedTo": ID, "content": content, - "to": [ap.AS_PUBLIC], - "cc": [f"{BASE_URL}/followers"], + "to": to, + "cc": cc, "published": published, "context": context, "conversation": context, diff --git a/app/main.py b/app/main.py index 562f061..4f33fc6 100644 --- a/app/main.py +++ b/app/main.py @@ -158,24 +158,30 @@ def index( request: Request, db: Session = Depends(get_db), _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker), + page: int | None = None, ) -> templates.TemplateResponse | ActivityPubResponse: if is_activitypub_requested(request): return ActivityPubResponse(LOCAL_ACTOR.ap_actor) + page = page or 1 + q = db.query(models.OutboxObject).filter( + models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC, + models.OutboxObject.is_deleted.is_(False), + models.OutboxObject.is_hidden_from_homepage.is_(False), + ) + total_count = q.count() + page_size = 2 + page_offset = (page - 1) * page_size + outbox_objects = ( - db.query(models.OutboxObject) - .options( + q.options( joinedload(models.OutboxObject.outbox_object_attachments).options( joinedload(models.OutboxObjectAttachment.upload) ) ) - .filter( - models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC, - models.OutboxObject.is_deleted.is_(False), - models.OutboxObject.is_hidden_from_homepage.is_(False), - ) .order_by(models.OutboxObject.ap_published_at.desc()) - .limit(20) + .offset(page_offset) + .limit(page_size) .all() ) @@ -183,7 +189,13 @@ def index( db, request, "index.html", - {"request": request, "objects": outbox_objects}, + { + "request": request, + "objects": outbox_objects, + "current_page": page, + "has_next_page": page_offset + len(outbox_objects) < total_count, + "has_previous_page": page > 1, + }, ) @@ -369,6 +381,33 @@ def outbox( ) +@app.get("/featured") +def featured( + db: Session = Depends(get_db), + _: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker), +) -> ActivityPubResponse: + outbox_objects = ( + db.query(models.OutboxObject) + .filter( + models.OutboxObject.visibility == ap.VisibilityEnum.PUBLIC, + models.OutboxObject.is_deleted.is_(False), + models.OutboxObject.is_pinned.is_(True), + ) + .order_by(models.OutboxObject.ap_published_at.desc()) + .limit(5) + .all() + ) + return ActivityPubResponse( + { + "@context": DEFAULT_CTX, + "id": f"{ID}/featured", + "type": "OrderedCollection", + "totalItems": len(outbox_objects), + "orderedItems": [ap.remove_context(a.ap_object) for a in outbox_objects], + } + ) + + @app.get("/o/{public_id}") def outbox_by_public_id( public_id: str, @@ -499,7 +538,10 @@ def post_remote_follow( @app.get("/.well-known/webfinger") def wellknown_webfinger(resource: str) -> JSONResponse: """Exposes/servers WebFinger data.""" + omg = f"acct:{USERNAME}@{DOMAIN}" + logger.info(f"{resource == omg}/{resource}/{omg}/{len(resource)}/{len(omg)}") if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: + logger.info(f"Got invalid req for {resource}") raise HTTPException(status_code=404) out = { @@ -651,6 +693,8 @@ def serve_proxy_media_resized( try: out = BytesIO(proxy_resp.content) i = Image.open(out) + if i.is_animated: + raise ValueError i.thumbnail((size, size)) resized_buf = BytesIO() i.save(resized_buf, format=i.format) @@ -660,6 +704,11 @@ def serve_proxy_media_resized( media_type=i.get_format_mimetype(), # type: ignore headers=proxy_resp_headers, ) + except ValueError: + return PlainTextResponse( + proxy_resp.content, + headers=proxy_resp_headers, + ) except Exception: logger.exception(f"Failed to resize {url} on the fly") return PlainTextResponse( diff --git a/app/models.py b/app/models.py index 89fb6c7..b4c0606 100644 --- a/app/models.py +++ b/app/models.py @@ -156,6 +156,9 @@ class OutboxObject(Base, BaseObject): og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True) + # For the featured collection + is_pinned = Column(Boolean, nullable=False, default=False) + # Never actually delete from the outbox is_deleted = Column(Boolean, nullable=False, default=False) @@ -181,6 +184,17 @@ class OutboxObject(Base, BaseObject): remote_side=id, uselist=False, ) + # For Follow activies + relates_to_actor_id = Column( + Integer, + ForeignKey("actor.id"), + nullable=True, + ) + relates_to_actor: Mapped[Optional["Actor"]] = relationship( + "Actor", + foreign_keys=[relates_to_actor_id], + uselist=False, + ) undone_by_outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True) diff --git a/app/scss/main.scss b/app/scss/main.scss index 4eb6d29..1744b8f 100644 --- a/app/scss/main.scss +++ b/app/scss/main.scss @@ -140,3 +140,6 @@ nav.flexbox { float: right; } } +.custom-emoji { + max-width: 25px; +} diff --git a/app/templates.py b/app/templates.py index 78fff44..ea82c52 100644 --- a/app/templates.py +++ b/app/templates.py @@ -163,11 +163,14 @@ def _update_inline_imgs(content): def _clean_html(html: str, note: Object) -> str: try: - return bleach.clean( - _replace_custom_emojis(_update_inline_imgs(highlight(html)), note), - tags=ALLOWED_TAGS, - attributes=ALLOWED_ATTRIBUTES, - strip=True, + return _replace_custom_emojis( + bleach.clean( + _update_inline_imgs(highlight(html)), + tags=ALLOWED_TAGS, + attributes=ALLOWED_ATTRIBUTES, + strip=True, + ), + note, ) except Exception: raise @@ -197,7 +200,7 @@ def _pluralize(count: int, singular: str = "", plural: str = "s") -> str: def _replace_custom_emojis(content: str, note: Object) -> str: idx = {} - for tag in note.ap_object.get("tag", []): + for tag in note.tags: if tag.get("type") == "Emoji": try: idx[tag["name"]] = proxied_media_url(tag["icon"]["url"]) diff --git a/app/templates/admin_new.html b/app/templates/admin_new.html index f11edea..4992960 100644 --- a/app/templates/admin_new.html +++ b/app/templates/admin_new.html @@ -10,7 +10,14 @@
{{ utils.embed_csrf_token() }} {{ utils.embed_redirect_url() }} - +

+ +

+

diff --git a/app/templates/admin_outbox.html b/app/templates/admin_outbox.html index c2b4a08..9f6a91b 100644 --- a/app/templates/admin_outbox.html +++ b/app/templates/admin_outbox.html @@ -2,15 +2,34 @@ {% extends "layout.html" %} {% block content %} +

Filter by +{% for ap_type in ["Note", "Like", "Announce", "Follow"] %} + + {% if request.query_params.filter_by == ap_type %} + {{ ap_type }} + {% else %} + {{ ap_type }} + {% endif %} +{% endfor %}. +{% if request.query_params.filter_by %}Reset filter{% endif %}

+

+ {% for outbox_object in outbox %} {% if outbox_object.ap_type == "Announce" %} +
You shared
{{ utils.display_object(outbox_object.relates_to_anybox_object) }} + {% elif outbox_object.ap_type == "Like" %} +
You liked
+ {{ utils.display_object(outbox_object.relates_to_anybox_object) }} + {% elif outbox_object.ap_type == "Follow" %} +
You followed
+ {{ utils.display_actor(outbox_object.relates_to_actor, actors_metadata) }} {% elif outbox_object.ap_type in ["Article", "Note", "Video"] %} {{ utils.display_object(outbox_object) }} -{% else %} - Implement {{ outbox_object.ap_type }} -{% endif %} + {% else %} + Implement {{ outbox_object.ap_type }} + {% endif %} {% endfor %} diff --git a/app/templates/header.html b/app/templates/header.html index cb07b94..bb6de21 100644 --- a/app/templates/header.html +++ b/app/templates/header.html @@ -24,6 +24,7 @@
  • {{ header_link("index", "Notes") }}
  • {{ header_link("followers", "Followers") }} {{ followers_count }}
  • {{ header_link("following", "Following") }} {{ following_count }}
    • +
  • {{ header_link("get_remote_follow", "Remote follow") }}
    • diff --git a/app/templates/index.html b/app/templates/index.html index 5b672dd..1f45032 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -7,6 +7,12 @@ {{ utils.display_object(outbox_object) }} {% endfor %} +{% if has_previous_page %} +Previous +{% endif %} +{% if has_next_page %} +Next +{% endif %} {% endblock %} diff --git a/app/templates/layout.html b/app/templates/layout.html index 0972101..fc60875 100644 --- a/app/templates/layout.html +++ b/app/templates/layout.html @@ -24,7 +24,6 @@
  • Admin
  • {{ admin_link("index", "Public") }}
  • {{ admin_link("admin_new", "New") }}
    • -
  • {{ admin_link("stream", "Stream") }}
  • {{ admin_link("admin_inbox", "Inbox") }}/{{ admin_link("admin_outbox", "Outbox") }}
  • {{ admin_link("get_notifications", "Notifications") }} {% if notifications_count %}({{ notifications_count }}){% endif %}
  • {{ admin_link("get_lookup", "Lookup") }}
    • diff --git a/app/templates/utils.html b/app/templates/utils.html index 9a1d770..564141b 100644 --- a/app/templates/utils.html +++ b/app/templates/utils.html @@ -42,6 +42,24 @@
    {% endmacro %} +{% macro admin_pin_button(ap_object_id) %} +
    + {{ embed_csrf_token() }} + {{ embed_redirect_url() }} + + +
    +{% endmacro %} + +{% macro admin_unpin_button(ap_object_id) %} +
    + {{ embed_csrf_token() }} + {{ embed_redirect_url() }} + + +
    +{% endmacro %} + {% macro admin_announce_button(ap_object_id) %}
    {{ embed_csrf_token() }} @@ -98,7 +116,7 @@ -
    {{ actor.name or actor.preferred_username }}
    +
    {{ actor.display_name | clean_html(actor) | safe }}
    {{ actor.handle }}
     @@ -156,7 +174,7 @@
    - {{ object.actor.name or object.actor.preferred_username }} + {{ object.actor.display_name }} {{ object.actor.handle }} {{ object.visibility.value }} @@ -206,8 +224,14 @@
    {{ admin_reply_button(object.ap_id) }}
    +
    + {% if object.is_pinned %} + {{ admin_unpin_button(object.ap_id) }} + {% else %} + {{ admin_pin_button(object.ap_id) }} + {% endif %} +
    {% endif %} - {% endif %} {% if object.is_from_inbox %}