From 98a2f8dbf85a70f6f69719c30fc93b9c5a3e7a05 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Jul 2019 22:05:28 +0200 Subject: [PATCH] Fix caching and tweak design --- activitypub.py | 1 + app.py | 33 +++++++++++++----- sass/base_theme.scss | 1 + templates/following.html | 2 +- templates/index.html | 6 ++-- templates/liked.html | 2 +- templates/lookup.html | 2 +- templates/stream.html | 12 +++---- utils/media.py | 74 ++++++++++++++++++++++++---------------- 9 files changed, 82 insertions(+), 51 deletions(-) diff --git a/activitypub.py b/activitypub.py index 0d4a5c2..15e64cb 100644 --- a/activitypub.py +++ b/activitypub.py @@ -569,6 +569,7 @@ class MicroblogPubBackend(Backend): ) return None + print(f"processing {create!r} and incrementing {in_reply_to}") creply = DB.activities.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, diff --git a/app.py b/app.py index 55e3de7..dbec07f 100644 --- a/app.py +++ b/app.py @@ -2639,6 +2639,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: if back.inbox_check_duplicate(MY_PERSON, activity.id): # The activity is already in the inbox app.logger.info(f"received duplicate activity {activity!r}, dropping it") + return back.save(Box.INBOX, activity) Tasks.process_new_activity(activity.id) @@ -2691,16 +2692,23 @@ def task_cache_attachments(): if actor.icon: MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + obj = None if activity.has_type(ap.ActivityType.CREATE): - for attachment in activity.get_object()._data.get("attachment", []): - if ( - attachment.get("mediaType", "").startswith("image/") - or attachment.get("type") == ap.ActivityType.IMAGE.value - ): - try: - MEDIA_CACHE.cache_attachment2(attachment["url"], iri) - except ValueError: - app.logger.exception(f"failed to cache {attachment}") + # This means a `Create` triggered the task + obj = activity.get_object() + elif activity.has_type(ap.CREATE_TYPES): + # This means a `Announce` triggered the task + obj = activity + else: + app.logger.warning(f"Don't know what to do with {activity!r}") + return + + # Iter the attachments + for attachment in obj._data.get("attachment", []): + try: + MEDIA_CACHE.cache_attachment2(attachment, iri) + except ValueError: + app.logger.exception(f"failed to cache {attachment}") app.logger.info(f"attachments cached for {iri}") @@ -2725,6 +2733,7 @@ def task_cache_actor() -> str: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") + # FIXME(tsileo): OG meta for Announce? if activity.has_type(ap.ActivityType.CREATE): Tasks.fetch_og_meta(iri) @@ -2766,6 +2775,10 @@ def task_cache_actor() -> str: app.logger.info(f"actor cached for {iri}") if also_cache_attachments and activity.has_type(ap.ActivityType.CREATE): Tasks.cache_attachments(iri) + elif also_cache_attachments and activity.has_type(ap.ActivityType.ANNOUNCE): + obj = activity.get_object() + Tasks.cache_attachments(obj.id) + Tasks.cache_actor(obj.id) except (ActivityGoneError, ActivityNotFoundError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) @@ -3201,6 +3214,8 @@ def task_cleanup_part_2(): MEDIA_CACHE.fs.delete(grid_item._id) DB.activities.delete_one({"_id": data["_id"]}) + # FIXME(tsileo): cleanup cache from announces object + p.push({}, "/task/cleanup_part_3") return "OK" diff --git a/sass/base_theme.scss b/sass/base_theme.scss index ce16014..c4c67b2 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -246,6 +246,7 @@ a:hover { color: $color-light; margin-right:5px; border-radius:2px; + float: left; } .bar-item-no-hover:hover { cursor: default; diff --git a/templates/following.html b/templates/following.html index e5874a8..c850f88 100644 --- a/templates/following.html +++ b/templates/following.html @@ -10,7 +10,7 @@
{% for (follow_id, follow) in following_data %} {% if session.logged_in %} -
+
diff --git a/templates/index.html b/templates/index.html index 13a488c..f1c9711 100644 --- a/templates/index.html +++ b/templates/index.html @@ -37,17 +37,17 @@ {% if "actor" in item.meta %} {% set boost_actor = item.meta.actor %} {% if session.logged_in %} -
- {{ boost_actor.name }} boosted +
+ {{ boost_actor.name }} boosted
{% else %} -

+

{{ boost_actor.name }} boosted

{% endif %} diff --git a/templates/liked.html b/templates/liked.html index 3548fd6..e9a6b8d 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -9,7 +9,7 @@
{% for item in liked %} {% if session.logged_in %} -
+
diff --git a/templates/lookup.html b/templates/lookup.html index 13675bf..52e7b20 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -16,7 +16,7 @@ {% set data = data.to_dict() %}
{% if data | has_actor_type %} -
+
diff --git a/templates/stream.html b/templates/stream.html index 71c25f4..d6d06e8 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -15,7 +15,7 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.meta.actor %} {% if boost_actor %} -

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

+

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

{% endif %} {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} @@ -24,26 +24,26 @@ {% if item | has_type('Like') %} {% set boost_actor = item.meta.actor %} -

{{ boost_actor.name or boost_actor.preferredUsername }} liked

+

{{ boost_actor.name or boost_actor.preferredUsername }} liked

{% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} {% endif %} {% endif %} {% if item | has_type('Follow') %} -

new follower +

new follower

{{ utils.display_actor_inline(item.meta.actor, size=50) }}
{% elif item | has_type('Accept') %} -

you started following

+

you started following

{{ utils.display_actor_inline(item.meta.actor, size=50) }}
{% elif item | has_type('Undo') %} -

unfollowed you

+

unfollowed you

{{ utils.display_actor_inline(item.meta.actor, size=50) }}
@@ -57,7 +57,7 @@ {% else %} {% if item | has_type('question_ended') %} -

poll ended

+

poll ended

{{ utils.display_note(item.activity.object) }} {% endif %} diff --git a/utils/media.py b/utils/media.py index d00aa67..aba132c 100644 --- a/utils/media.py +++ b/utils/media.py @@ -4,6 +4,9 @@ from enum import Enum from gzip import GzipFile from io import BytesIO from typing import Any +from typing import Dict + +from little_boxes import activitypub as ap import gridfs import piexif @@ -115,45 +118,56 @@ class MediaCache(object): ) return - def cache_attachment2(self, url: str, remote_id: str) -> None: + def cache_attachment2(self, attachment: Dict[str, Any], remote_id: str) -> None: + url = attachment["url"] + + # Ensure it's not already there if self.fs.find_one({"url": url, "kind": Kind.ATTACHMENT.value}): return + + # If it's an image, make some thumbnails if ( url.endswith(".png") or url.endswith(".jpg") or url.endswith(".jpeg") or url.endswith(".gif") + or attachment.get("mediaType", "").startswith("image/") + or ap._has_type(attachment.get("type"), ap.ActivityType.IMAGE) ): - i = load(url, self.user_agent) - # Save the original attachment (gzipped) - with BytesIO() as buf: - f1 = GzipFile(mode="wb", fileobj=buf) - i.save(f1, format=i.format) - f1.close() - buf.seek(0) - self.fs.put( - buf, - url=url, - size=None, - content_type=i.get_format_mimetype(), - kind=Kind.ATTACHMENT.value, - remote_id=remote_id, - ) - # Save a thumbnail (gzipped) - i.thumbnail((720, 720)) - with BytesIO() as buf: - with GzipFile(mode="wb", fileobj=buf) as f1: + try: + i = load(url, self.user_agent) + # Save the original attachment (gzipped) + with BytesIO() as buf: + f1 = GzipFile(mode="wb", fileobj=buf) i.save(f1, format=i.format) - buf.seek(0) - self.fs.put( - buf, - url=url, - size=720, - content_type=i.get_format_mimetype(), - kind=Kind.ATTACHMENT.value, - remote_id=remote_id, - ) - return + f1.close() + buf.seek(0) + self.fs.put( + buf, + url=url, + size=None, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + remote_id=remote_id, + ) + # Save a thumbnail (gzipped) + i.thumbnail((720, 720)) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as f1: + i.save(f1, format=i.format) + buf.seek(0) + self.fs.put( + buf, + url=url, + size=720, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + remote_id=remote_id, + ) + return + except Exception: + # FIXME(tsileo): logging + pass # The attachment is not an image, download and save it anyway with requests.get(