From 3e7b09b7a854b0f2b21f3b81d43b3f6d1f362ef3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 20 Jul 2018 10:56:39 +0200 Subject: [PATCH] More caching --- activitypub.py | 33 ++++++++++++++++++++++++++++++--- app.py | 25 +++++++++++++++++++++++++ tasks.py | 13 +------------ templates/index.html | 4 ++-- templates/liked.html | 2 +- templates/stream.html | 10 +++++----- 6 files changed, 64 insertions(+), 23 deletions(-) diff --git a/activitypub.py b/activitypub.py index 83e58ab..9a1d0b8 100644 --- a/activitypub.py +++ b/activitypub.py @@ -32,6 +32,15 @@ logger = logging.getLogger(__name__) ACTORS_CACHE = LRUCache(maxsize=256) +def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: + return { + "url": actor.url, + "icon": actor.icon, + "name": actor.name, + "preferredUsername": actor.preferredUsername, + } + + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() @@ -263,8 +272,15 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}, "$set": {"meta.liked": like.id}}, ) + DB.activities.update_one( - {"remote_id": like.id}, {"$set": {"meta.object": obj.to_dict(embed=True)}} + {"remote_id": like.id}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) @ensure_it_is_me @@ -291,7 +307,12 @@ class MicroblogPubBackend(Backend): obj = announce.get_object() DB.activities.update_one( {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) DB.activities.update_one( {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": 1}} @@ -313,8 +334,14 @@ class MicroblogPubBackend(Backend): obj = announce.get_object() DB.activities.update_one( {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} ) diff --git a/app.py b/app.py index 50291d3..00a290a 100644 --- a/app.py +++ b/app.py @@ -679,6 +679,31 @@ def tmp_migrate4(): def tmp_migrate5(): for activity in DB.activities.find(): tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + + return "Done" + + +@app.route("/migration5") +@login_required +def tmp_migrate6(): + for activity in DB.activities.find(): + # tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + try: + a = ap.parse_activity(activity["activity"]) + if a.has_type([ActivityType.LIKE, ActivityType.FOLLOW]): + DB.activities.update_one( + {"remote_id": a.id}, + { + "$set": { + "meta.object_actor": activitypub._actor_to_meta( + a.get_object().get_actor() + ) + } + }, + ) + except Exception: + app.logger.exception(f"processing {activity} failed") + return "Done" diff --git a/tasks.py b/tasks.py index 3eccb3e..f348ded 100644 --- a/tasks.py +++ b/tasks.py @@ -2,8 +2,6 @@ import json import logging import os import random -from typing import Dict -from typing import Any import requests from celery import Celery @@ -64,15 +62,6 @@ def process_new_activity(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: - return { - "url": actor.url, - "icon": actor.icon, - "name": actor.name, - "preferredUsername": actor.preferredUsername, - } - - @app.task(bind=True, max_retries=12) def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: @@ -83,7 +72,7 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: # Cache the actor info DB.activities.update_one( - {"remote_id": iri}, {"$set": {"meta.actor": _actor_to_meta(actor)}} + {"remote_id": iri}, {"$set": {"meta.actor": activitypub._actor_to_meta(actor)}} ) log.info(f"actor cached for {iri}") diff --git a/templates/index.html b/templates/index.html index a398087..55e64a5 100644 --- a/templates/index.html +++ b/templates/index.html @@ -24,7 +24,7 @@ {% for item in outbox_data %} {% if item | has_type('Announce') %} - {% set boost_actor = item.activity.actor | get_actor %} + {% set boost_actor = item.meta.actor %} {% if session.logged_in %}
{{ boost_actor.name }} boosted @@ -41,7 +41,7 @@

{% endif %} {% if item.meta.object %} - {{ utils.display_note(item.meta.object, ui=False) }} + {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} {% endif %} {% elif item | has_type('Create') %} {{ utils.display_note(item.activity.object, meta=item.meta, no_color=True) }} diff --git a/templates/liked.html b/templates/liked.html index 29b903f..3548fd6 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -20,7 +20,7 @@ {% endif %} {% if item.meta.object %} - {{ utils.display_note(item.meta.object) }} + {{ utils.display_note(item.meta.object, meta={'actor': item.meta.object_actor}) }} {% endif %} {% endfor %} diff --git a/templates/stream.html b/templates/stream.html index 365288a..f12c0c7 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -1,6 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} -{% block title %}Stream - {{ config.NAME }}{% endblock %} +{% block title %}{% if request.path == url_for('admin_stream') %}Stream{% else %}Notifications{% endif %} - {{ config.NAME }}{% endblock %} {% block content %}
{% include "header.html" %} @@ -13,7 +13,7 @@ {% else %} {% if item | has_type('Announce') %} - {% set boost_actor = item.activity.actor | get_actor %} + {% set boost_actor = item.meta.actor %}

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

{% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} @@ -23,19 +23,19 @@ {% if item | has_type('Follow') %}

new follower

- {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
{% elif item | has_type('Accept') %}

you started following

- {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
{% elif item | has_type('Undo') %}

unfollowed you

- {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
{% else %}