From dfe7d692e3e05dabac5afe939b595b8ba915d1d9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 7 Jul 2022 20:37:16 +0200 Subject: [PATCH] Bugfixes and cleanup --- ...b5bcc918ce_tweak_inbox_objects_metadata.py | 34 +++++ app/admin.py | 79 +++++++++- app/ap_object.py | 14 +- app/boxes.py | 142 ++++++++++-------- app/config.py | 2 +- app/models.py | 6 +- app/scss/main.scss | 11 ++ app/templates/admin_inbox.html | 4 +- app/templates/layout.html | 1 + app/templates/utils.html | 4 +- scripts/config_wizard.py | 4 +- 11 files changed, 227 insertions(+), 74 deletions(-) create mode 100644 alembic/versions/79b5bcc918ce_tweak_inbox_objects_metadata.py diff --git a/alembic/versions/79b5bcc918ce_tweak_inbox_objects_metadata.py b/alembic/versions/79b5bcc918ce_tweak_inbox_objects_metadata.py new file mode 100644 index 0000000..1ea5e22 --- /dev/null +++ b/alembic/versions/79b5bcc918ce_tweak_inbox_objects_metadata.py @@ -0,0 +1,34 @@ +"""Tweak inbox objects metadata + +Revision ID: 79b5bcc918ce +Revises: 93e36ff5c691 +Create Date: 2022-07-07 18:03:46.945044 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = '79b5bcc918ce' +down_revision = '93e36ff5c691' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('inbox', sa.Column('is_deleted', sa.Boolean(), nullable=False, server_default="0")) + op.add_column('inbox', sa.Column('replies_count', sa.Integer(), nullable=False, server_default="0")) + op.drop_column('inbox', 'has_replies') + op.create_index(op.f('ix_outgoing_activity_id'), 'outgoing_activity', ['id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_outgoing_activity_id'), table_name='outgoing_activity') + op.add_column('inbox', sa.Column('has_replies', sa.BOOLEAN(), nullable=False)) + op.drop_column('inbox', 'replies_count') + op.drop_column('inbox', 'is_deleted') + # ### end Alembic commands ### diff --git a/app/admin.py b/app/admin.py index e453706..4b0fd1e 100644 --- a/app/admin.py +++ b/app/admin.py @@ -172,6 +172,7 @@ async def admin_bookmarks( ["Note", "Article", "Video", "Announce"] ), models.InboxObject.is_bookmarked.is_(True), + models.InboxObject.is_deleted.is_(False), ) .options( joinedload(models.InboxObject.relates_to_inbox_object), @@ -199,6 +200,77 @@ async def admin_bookmarks( ) +@router.get("/stream") +async def admin_stream( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + cursor: str | None = None, +) -> templates.TemplateResponse: + where = [ + models.InboxObject.is_hidden_from_stream.is_(False), + models.InboxObject.is_deleted.is_(False), + ] + if cursor: + where.append( + models.InboxObject.ap_published_at < pagination.decode_cursor(cursor) + ) + + page_size = 20 + remaining_count = await db_session.scalar( + select(func.count(models.InboxObject.id)).where(*where) + ) + q = select(models.InboxObject).where(*where) + + inbox = ( + ( + await db_session.scalars( + q.options( + joinedload(models.InboxObject.relates_to_inbox_object).options( + joinedload(models.InboxObject.actor) + ), + joinedload(models.InboxObject.relates_to_outbox_object).options( + joinedload( + models.OutboxObject.outbox_object_attachments + ).options(joinedload(models.OutboxObjectAttachment.upload)), + ), + joinedload(models.InboxObject.actor), + ) + .order_by(models.InboxObject.ap_published_at.desc()) + .limit(20) + ) + ) + .unique() + .all() + ) + + next_cursor = ( + pagination.encode_cursor(inbox[-1].ap_published_at) + if inbox and remaining_count > page_size + else None + ) + + actors_metadata = await get_actors_metadata( + db_session, + [ + inbox_object.actor + for inbox_object in inbox + if inbox_object.ap_type == "Follow" + ], + ) + + return await templates.render_template( + db_session, + request, + "admin_inbox.html", + { + "inbox": inbox, + "actors_metadata": actors_metadata, + "next_cursor": next_cursor, + "show_filters": False, + }, + ) + + @router.get("/inbox") async def admin_inbox( request: Request, @@ -207,7 +279,10 @@ async def admin_inbox( cursor: str | None = None, ) -> templates.TemplateResponse: where = [ - models.InboxObject.ap_type.not_in(["Accept", "Delete", "Create", "Update"]) + models.InboxObject.ap_type.not_in( + ["Accept", "Delete", "Create", "Update", "Undo"] + ), + models.InboxObject.is_deleted.is_(False), ] if filter_by: where.append(models.InboxObject.ap_type == filter_by) @@ -267,6 +342,7 @@ async def admin_inbox( "inbox": inbox, "actors_metadata": actors_metadata, "next_cursor": next_cursor, + "show_filters": True, }, ) @@ -425,6 +501,7 @@ async def admin_profile( await db_session.scalars( select(models.InboxObject) .where( + models.InboxObject.is_deleted.is_(False), models.InboxObject.actor_id == actor.id, models.InboxObject.ap_type.in_(["Note", "Article", "Video"]), ) diff --git a/app/ap_object.py b/app/ap_object.py index f715544..124f02a 100644 --- a/app/ap_object.py +++ b/app/ap_object.py @@ -103,7 +103,19 @@ class Object: ) ) break - + elif link.get("mediaType", "") == "application/x-mpegURL": + for tag in ap.as_list(link.get("tag", [])): + if tag.get("mediaType", "").startswith("video"): + proxied_url = proxied_media_url(tag["href"]) + attachments.append( + Attachment( + type="Video", + mediaType=tag["mediaType"], + url=tag["href"], + proxiedUrl=proxied_url, + ) + ) + break return attachments @property diff --git a/app/boxes.py b/app/boxes.py index be88b32..5705a58 100644 --- a/app/boxes.py +++ b/app/boxes.py @@ -517,11 +517,8 @@ async def _handle_delete_activity( inbox_object_id=delete_activity.id, ) - # TODO(ts): do we need to delete related activities? should we keep - # bookmarked objects with a deleted flag? logger.info(f"Deleting {ap_object_to_delete.ap_type}/{ap_object_to_delete.ap_id}") - await db_session.delete(ap_object_to_delete) - await db_session.flush() + ap_object_to_delete.is_deleted = True async def _handle_follow_follow_activity( @@ -575,6 +572,7 @@ async def _handle_undo_activity( return ap_activity_to_undo.undone_by_inbox_object_id = undo_activity.id + ap_activity_to_undo.is_deleted = True if ap_activity_to_undo.ap_type == "Follow": logger.info(f"Undo follow from {from_actor.ap_id}") @@ -635,8 +633,6 @@ async def _handle_undo_activity( inbox_object_id=ap_activity_to_undo.id, ) db_session.add(notif) - - # FIXME(ts): what to do with ap_activity_to_undo? flag? delete? else: logger.warning(f"Don't know how to undo {ap_activity_to_undo.ap_type} activity") @@ -687,6 +683,14 @@ async def _handle_create_activity( if "published" in ro.ap_object: ap_published_at = isoparse(ro.ap_object["published"]) + is_reply = bool(ro.in_reply_to) + is_local_reply = ro.in_reply_to and ro.in_reply_to.startswith(BASE_URL) + is_mention = False + tags = ro.ap_object.get("tag", []) + for tag in tags: + if tag.get("name") == LOCAL_ACTOR.handle or tag.get("href") == LOCAL_ACTOR.url: + is_mention = True + inbox_object = models.InboxObject( server=urlparse(ro.ap_id).netloc, actor_id=from_actor.id, @@ -701,11 +705,7 @@ async def _handle_create_activity( relates_to_outbox_object_id=None, activity_object_ap_id=ro.activity_object_ap_id, # Hide replies from the stream - is_hidden_from_stream=( - True - if (ro.in_reply_to and not ro.in_reply_to.startswith(BASE_URL)) - else False - ), # TODO: handle mentions + is_hidden_from_stream=not (not is_reply or is_mention or is_local_reply), ) db_session.add(inbox_object) @@ -714,28 +714,35 @@ async def _handle_create_activity( create_activity.relates_to_inbox_object_id = inbox_object.id - tags = inbox_object.ap_object.get("tag") - - if not tags: - logger.info("No tags to process") - return None - - if not isinstance(tags, list): - logger.info(f"Invalid tags: {tags}") - return None - - if inbox_object.in_reply_to and inbox_object.in_reply_to.startswith(BASE_URL): - await db_session.execute( - update(models.OutboxObject) - .where( - models.OutboxObject.ap_id == inbox_object.in_reply_to, - ) - .values(replies_count=models.OutboxObject.replies_count + 1) + if inbox_object.in_reply_to: + replied_object = await get_anybox_object_by_ap_id( + db_session, inbox_object.in_reply_to ) + if replied_object: + if replied_object.is_from_outbox: + await db_session.execute( + update(models.OutboxObject) + .where( + models.OutboxObject.id == replied_object.id, + ) + .values(replies_count=models.OutboxObject.replies_count + 1) + ) + else: + await db_session.execute( + update(models.InboxObject) + .where( + models.InboxObject.id == replied_object.id, + ) + .values(replies_count=models.InboxObject.replies_count + 1) + ) # This object is a reply of a local object, we may need to forward it # to our followers (we can only forward JSON-LD signed activities) - if create_activity.has_ld_signature: + if ( + replied_object + and replied_object.is_from_outbox + and create_activity.has_ld_signature + ): logger.info("Forwarding Create activity as it's a local reply") recipients = await _get_followers_recipients(db_session) for rcp in recipients: @@ -746,14 +753,13 @@ async def _handle_create_activity( inbox_object_id=create_activity.id, ) - for tag in tags: - if tag.get("name") == LOCAL_ACTOR.handle or tag.get("href") == LOCAL_ACTOR.url: - notif = models.Notification( - notification_type=models.NotificationType.MENTION, - actor_id=from_actor.id, - inbox_object_id=inbox_object.id, - ) - db_session.add(notif) + if is_mention: + notif = models.Notification( + notification_type=models.NotificationType.MENTION, + actor_id=from_actor.id, + inbox_object_id=inbox_object.id, + ) + db_session.add(notif) async def save_to_inbox( @@ -825,7 +831,6 @@ async def save_to_inbox( if relates_to_outbox_object else None, activity_object_ap_id=activity_ro.activity_object_ap_id, - # Hide replies from the stream is_hidden_from_stream=True, ) @@ -921,7 +926,9 @@ async def save_to_inbox( else: # This is announce for a maybe unknown object if relates_to_inbox_object: - logger.info("Nothing to do, we already know about this object") + # We already know about this object, show the announce in the + # streal + inbox_object.is_hidden_from_stream = False else: # Save it as an inbox object if not activity_ro.activity_object_ap_id: @@ -946,6 +953,7 @@ async def save_to_inbox( db_session.add(announced_inbox_object) await db_session.flush() inbox_object.relates_to_inbox_object_id = announced_inbox_object.id + inbox_object.is_hidden_from_stream = False elif activity_ro.ap_type in ["Like", "Announce"]: if not relates_to_outbox_object: logger.info( @@ -1030,40 +1038,44 @@ async def get_replies_tree( db_session: AsyncSession, requested_object: AnyboxObject, ) -> ReplyTreeNode: - # TODO: handle visibility + # XXX: PeerTube video don't use context tree_nodes: list[AnyboxObject] = [] - tree_nodes.extend( - ( - await db_session.scalars( - select(models.InboxObject) - .where( - models.InboxObject.ap_context == requested_object.ap_context, - models.InboxObject.ap_type.not_in(["Announce"]), + if requested_object.ap_context is None: + tree_nodes = [requested_object] + else: + # TODO: handle visibility + tree_nodes.extend( + ( + await db_session.scalars( + select(models.InboxObject) + .where( + models.InboxObject.ap_context == requested_object.ap_context, + models.InboxObject.ap_type.not_in(["Announce"]), + ) + .options(joinedload(models.InboxObject.actor)) ) - .options(joinedload(models.InboxObject.actor)) ) + .unique() + .all() ) - .unique() - .all() - ) - tree_nodes.extend( - ( - await db_session.scalars( - select(models.OutboxObject) - .where( - models.OutboxObject.ap_context == requested_object.ap_context, - models.OutboxObject.is_deleted.is_(False), - ) - .options( - joinedload(models.OutboxObject.outbox_object_attachments).options( - joinedload(models.OutboxObjectAttachment.upload) + tree_nodes.extend( + ( + await db_session.scalars( + select(models.OutboxObject) + .where( + models.OutboxObject.ap_context == requested_object.ap_context, + models.OutboxObject.is_deleted.is_(False), + ) + .options( + joinedload( + models.OutboxObject.outbox_object_attachments + ).options(joinedload(models.OutboxObjectAttachment.upload)) ) ) ) + .unique() + .all() ) - .unique() - .all() - ) nodes_by_in_reply_to = defaultdict(list) for node in tree_nodes: nodes_by_in_reply_to[node.in_reply_to].append(node) diff --git a/app/config.py b/app/config.py index 9d9a143..17c18d0 100644 --- a/app/config.py +++ b/app/config.py @@ -15,7 +15,7 @@ from app.utils.emoji import _load_emojis ROOT_DIR = Path().parent.resolve() -_CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml") +_CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "profile.toml") VERSION_COMMIT = ( subprocess.check_output(["git", "rev-parse", "--short=8", "HEAD"]) diff --git a/app/models.py b/app/models.py index 69700da..379377e 100644 --- a/app/models.py +++ b/app/models.py @@ -100,8 +100,10 @@ class InboxObject(Base, BaseObject): is_bookmarked = Column(Boolean, nullable=False, default=False) - # FIXME(ts): do we need this? - has_replies = Column(Boolean, nullable=False, default=False) + # Used to mark deleted objects, but also activities that were undone + is_deleted = Column(Boolean, nullable=False, default=False) + + replies_count = Column(Integer, nullable=False, default=0) og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True) diff --git a/app/scss/main.scss b/app/scss/main.scss index 634067c..0f4e627 100644 --- a/app/scss/main.scss +++ b/app/scss/main.scss @@ -174,6 +174,17 @@ nav.flexbox { } } +.activity-expanded { + .activity-attachment { +img, audio, video { + width: 100%; + max-width: 740px; + margin: 30px 0; + } + + } +} + .activity-wrap { margin: 0 auto; diff --git a/app/templates/admin_inbox.html b/app/templates/admin_inbox.html index dff7bd7..c4f0f9d 100644 --- a/app/templates/admin_inbox.html +++ b/app/templates/admin_inbox.html @@ -2,7 +2,9 @@ {% extends "layout.html" %} {% block content %} +{% if show_filters %} {{ utils.display_box_filters("admin_inbox") }} +{% endif %} {% macro actor_action(inbox_object, text) %}
@@ -33,7 +35,7 @@ {% endfor %} {% if next_cursor %} -

See more

+

See more

{% endif %} {% endblock %} diff --git a/app/templates/layout.html b/app/templates/layout.html index 8fcc18f..09cb128 100644 --- a/app/templates/layout.html +++ b/app/templates/layout.html @@ -28,6 +28,7 @@
  • Admin
  • {{ admin_link("index", "Public") }}
  • {{ admin_link("admin_new", "New") }}
    • +
  • {{ admin_link("admin_stream", "Stream") }}
  • {{ admin_link("admin_inbox", "Inbox") }}/{{ admin_link("admin_outbox", "Outbox") }}
  • {{ admin_link("get_notifications", "Notifications") }} {% if notifications_count %}({{ notifications_count }}){% endif %}
  • {{ admin_link("get_lookup", "Lookup") }}
    • diff --git a/app/templates/utils.html b/app/templates/utils.html index 5edb0d0..f297639 100644 --- a/app/templates/utils.html +++ b/app/templates/utils.html @@ -210,7 +210,7 @@ {% if attachment.type == "Image" or (attachment | has_media_type("image")) %} {{ attachment.name }} {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %} - + {% elif attachment.type == "Audio" or (attachment | has_media_type("audio")) %} {% else %} @@ -264,7 +264,9 @@ +
    {{ display_attachments(object) }} +
    {% if likes or shares %}
    diff --git a/scripts/config_wizard.py b/scripts/config_wizard.py index 3a8bdaa..192b0a2 100644 --- a/scripts/config_wizard.py +++ b/scripts/config_wizard.py @@ -29,11 +29,11 @@ def main() -> None: else: generate_key(_KEY_PATH) - config_file = Path("data/me.toml") + config_file = Path("data/profile.toml") if config_file.exists(): # Spit out the relative path for the "config artifacts" - rconfig_file = "data/me.toml" + rconfig_file = "data/profile.toml" print( f"Existing setup detected, please delete {rconfig_file} " "before restarting the wizard"