From eb4d795ede595728c99283681e2b9f0077e539bc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 29 Jun 2018 22:16:26 +0200 Subject: [PATCH] HUGE cleanup --- activitypub.py | 207 ++++++++++------------ app.py | 402 +++++++++++++++++++++--------------------- templates/index.html | 13 +- templates/liked.html | 18 ++ templates/stream.html | 30 ++-- templates/tags.html | 2 +- templates/utils.html | 80 ++++----- 7 files changed, 370 insertions(+), 382 deletions(-) create mode 100644 templates/liked.html diff --git a/activitypub.py b/activitypub.py index e27dfd7..b891674 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,6 +1,7 @@ import logging import os from datetime import datetime +from enum import Enum from typing import Any from typing import Dict from typing import List @@ -17,12 +18,12 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME -from little_boxes import strtobool from little_boxes import activitypub as ap +from little_boxes import strtobool +from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error -from little_boxes.activitypub import _to_list logger = logging.getLogger(__name__) @@ -46,6 +47,12 @@ def ensure_it_is_me(f): return wrapper +class Box(Enum): + INBOX = "inbox" + OUTBOX = "outbox" + REPLIES = "replies" + + class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" @@ -68,10 +75,11 @@ class MicroblogPubBackend(Backend): """URL for activity link.""" return f"{BASE_URL}/note/{obj_id}" - @ensure_it_is_me - def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - DB.outbox.insert_one( + def save(self, box: Box, activity: ap.BaseActivity) -> None: + """Custom helper for saving an activity to the DB.""" + DB.activities.insert_one( { + "box": box.value, "activity": activity.to_dict(), "type": _to_list(activity.type), "remote_id": activity.id, @@ -79,11 +87,16 @@ class MicroblogPubBackend(Backend): } ) + @ensure_it_is_me + def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + self.save(Box.OUTBOX, activity) + @ensure_it_is_me def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: return bool( - DB.outbox.find_one( + DB.activities.find_one( { + "box": Box.OUTBOX.value, "type": ap.ActivityType.BLOCK.value, "activity.object": actor_id, "meta.undo": False, @@ -101,14 +114,14 @@ class MicroblogPubBackend(Backend): if iri.endswith("/activity"): iri = iri.replace("/activity", "") is_a_note = True - data = DB.outbox.find_one({"remote_id": iri}) - if data: - if is_a_note: - return data["activity"]["object"] + data = DB.activities.find_one({"box": Box.OUTBOX.value, "remote_id": iri}) + if data and is_a_note: + return data["activity"]["object"] + elif data: return data["activity"] else: # Check if the activity is stored in the inbox - data = DB.inbox.find_one({"remote_id": iri}) + data = DB.activities.find_one({"remote_id": iri}) if data: return data["activity"] @@ -117,18 +130,11 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: - return bool(DB.inbox.find_one({"remote_id": iri})) + return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) @ensure_it_is_me def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - DB.inbox.insert_one( - { - "activity": activity.to_dict(), - "type": _to_list(activity.type), - "remote_id": activity.id, - "meta": {"undo": False, "deleted": False}, - } - ) + self.save(Box.INBOX, activity) @ensure_it_is_me def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: @@ -161,41 +167,37 @@ class MicroblogPubBackend(Backend): def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + DB.activities.update_one( + {"box": Box.OUTBOX.value, "activity.object.id": obj.id}, + {"$inc": {"meta.count_like": 1}}, ) @ensure_it_is_me def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + DB.activities.update_one( + {"box": Box.OUTBOX.value, "activity.object.id": obj.id}, + {"$inc": {"meta.count_like": -1}}, ) @ensure_it_is_me def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + DB.activities.update_one( + {"activity.object.id": obj.id}, + {"$inc": {"meta.count_like": 1}, "$set": {"meta.liked": like.id}}, ) - - # Keep track of the like we just performed - DB.inbox.update_one( - {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} + DB.activities.update_one( + {"remote_id": like.id}, {"$set": {"meta.object": obj.to_dict(embed=True)}} ) @ensure_it_is_me def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} - ) - - DB.inbox.update_one( - {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} + DB.activities.update_one( + {"activity.object.id": obj.id}, + {"$inc": {"meta.count_like": -1}, "$set": {"meta.liked": False}}, ) @ensure_it_is_me @@ -204,57 +206,57 @@ class MicroblogPubBackend(Backend): "object" ].startswith("http"): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + # or remote it? logger.warn( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - # FIXME(tsileo): Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(announce._data["object"], str): - obj_iri = announce._data["object"] - else: - obj_iri = self.get_object().id - DB.outbox.update_one( - {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + DB.activities.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": 1}} ) @ensure_it_is_me def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} ) @ensure_it_is_me def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() - DB.inbox.update_one( + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} ) @ensure_it_is_me def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() - DB.inbox.update_one( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} ) @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: - if not DB.inbox.find_one_and_update( - {"activity.object.id": delete.get_object().id}, - {"$set": {"meta.deleted": True}}, - ): - DB.threads.update_one( - {"activity.object.id": delete.get_object().id}, - {"$set": {"meta.deleted": True}}, - ) - obj = delete.get_object() + DB.activities.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.deleted": True}} + ) + if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = ap.parse_activity( - DB.inbox.find_one( + DB.activities.find_one( { "activity.object.id": delete.get_object().id, "type": ap.ActivityType.CREATE.value, @@ -268,14 +270,14 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: - DB.outbox.update_one( + DB.activities.update_one( {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, ) obj = delete.get_object() if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = ap.parse_activity( - DB.outbox.find_one( + DB.activities.find_one( { "activity.object.id": delete.get_object().id, "type": ap.ActivityType.CREATE.value, @@ -289,15 +291,10 @@ class MicroblogPubBackend(Backend): def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: - if not DB.inbox.find_one_and_update( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"activity.object": obj.to_dict()}}, - ): - DB.threads.update_one( - {"activity.object.id": obj.id}, - {"$set": {"activity.object": obj.to_dict()}}, - ) - + ) # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor @ensure_it_is_me @@ -322,7 +319,7 @@ class MicroblogPubBackend(Backend): print(f"updating note from outbox {obj!r} {update}") logger.info(f"updating note from outbox {obj!r} {update}") - DB.outbox.update_one({"activity.object.id": obj["id"]}, update) + DB.activities.update_one({"activity.object.id": obj["id"]}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) @@ -340,18 +337,10 @@ class MicroblogPubBackend(Backend): if not in_reply_to: pass - if not DB.inbox.find_one_and_update( + DB.activities.update_one( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ): - if not DB.outbox.find_one_and_update( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ): - DB.threads.update_one( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ) + ) @ensure_it_is_me def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: @@ -365,24 +354,14 @@ class MicroblogPubBackend(Backend): root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) - if not DB.inbox.find_one_and_update( + creply = DB.activities.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, - ): - if not DB.outbox.find_one_and_update( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, - ): - # It means the activity is not in the inbox, and not in the outbox, we want to save it - DB.threads.insert_one( - { - "activity": reply.to_dict(), - "type": _to_list(reply.type), - "remote_id": reply.id, - "meta": {"undo": False, "deleted": False}, - } - ) - new_threads.append(reply.id) + ) + if not creply: + # It means the activity is not in the inbox, and not in the outbox, we want to save it + self.save(Box.REPLIES, reply) + new_threads.append(reply.id) while reply is not None: in_reply_to = reply.inReplyTo @@ -391,25 +370,15 @@ class MicroblogPubBackend(Backend): root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) q = {"activity.object.id": root_reply} - if not DB.inbox.count(q) and not DB.outbox.count(q): - DB.threads.insert_one( - { - "activity": reply.to_dict(), - "type": _to_list(reply.type), - "remote_id": reply.id, - "meta": {"undo": False, "deleted": False}, - } - ) + if not DB.activities.count(q): + self.save(Box.REPLIES, reply) new_threads.append(reply.id) - q = {"remote_id": create.id} - if not DB.inbox.find_one_and_update( - q, {"$set": {"meta.thread_root_parent": root_reply}} - ): - DB.outbox.update_one(q, {"$set": {"meta.thread_root_parent": root_reply}}) - - DB.threads.update( - {"remote_id": {"$in": new_threads}}, + DB.activities.update_one( + {"remote_id": create.id}, {"$set": {"meta.thread_root_parent": root_reply}} + ) + DB.activities.update( + {"box": Box.REPLIES.value, "remote_id": {"$in": new_threads}}, {"$set": {"meta.thread_root_parent": root_reply}}, ) @@ -423,7 +392,9 @@ def gen_feed(): fg.description(f"{USERNAME} notes") fg.logo(ME.get("icon", {}).get("url")) fg.language("en") - for item in DB.outbox.find({"type": "Create"}, limit=50): + for item in DB.activities.find( + {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 + ): fe = fg.add_entry() fe.id(item["activity"]["object"].get("url")) fe.link(href=item["activity"]["object"].get("url")) @@ -435,7 +406,9 @@ def gen_feed(): def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] - for item in DB.outbox.find({"type": "Create"}, limit=50): + for item in DB.activities.find( + {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 + ): data.append( { "id": item["id"], @@ -471,11 +444,15 @@ def build_inbox_json_feed( data = [] cursor = None - q: Dict[str, Any] = {"type": "Create", "meta.deleted": False} + q: Dict[str, Any] = { + "type": "Create", + "meta.deleted": False, + "box": Box.INBOX.value, + } if request_cursor: q["_id"] = {"$lt": request_cursor} - for item in DB.inbox.find(q, limit=50).sort("_id", -1): + for item in DB.activities.find(q, limit=50).sort("_id", -1): actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) data.append( { diff --git a/app.py b/app.py index 3b947dc..2cb4a27 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ from werkzeug.utils import secure_filename import activitypub import config +from activitypub import Box from activitypub import embed_collection from config import ADMIN_API_KEY from config import BASE_URL @@ -56,6 +57,7 @@ from config import _drop_db from config import custom_cache_purge_hook from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import _to_list from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown @@ -111,18 +113,21 @@ def inject_config(): "activity.object.inReplyTo": None, "meta.deleted": False, } - notes_count = DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]} + notes_count = DB.activities.find( + {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - with_replies_count = DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]} + with_replies_count = DB.activities.find( + {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() - liked_count = DB.outbox.count({ - "meta.deleted": False, - "meta.undo": False, - "type": ActivityType.LIKE.value, - }) + liked_count = DB.activities.count( + { + "box": Box.OUTBOX.value, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.LIKE.value, + } + ) return dict( microblogpub_version=VERSION, config=config, @@ -132,6 +137,7 @@ def inject_config(): notes_count=notes_count, liked_count=liked_count, with_replies_count=with_replies_count, + me=ME, ) @@ -172,6 +178,11 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) +@app.template_filter() +def permalink_id(val): + return str(hash(val)) + + @app.template_filter() def quote_plus(t): return urllib.parse.quote_plus(t) @@ -221,6 +232,13 @@ def format_timeago(val): return val +@app.template_filter() +def has_type(doc, _type): + if _type in _to_list(doc["type"]): + return True + return False + + def _is_img(filename): filename = filename.lower() if ( @@ -370,9 +388,7 @@ def login(): payload = u2f.begin_authentication(ID, devices) session["challenge"] = payload - return render_template( - "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload - ) + return render_template("login.html", u2f_enabled=u2f_enabled, payload=payload) @app.route("/remote_follow", methods=["GET", "POST"]) @@ -429,6 +445,45 @@ def u2f_register(): # Activity pub routes +@app.route("/migration1_step1") +@login_required +def tmp_migrate(): + for activity in DB.outbox.find(): + activity["box"] = Box.OUTBOX.value + DB.activities.insert_one(activity) + for activity in DB.inbox.find(): + activity["box"] = Box.INBOX.value + DB.activities.insert_one(activity) + for activity in DB.replies.find(): + activity["box"] = Box.REPLIES.value + DB.activities.insert_one(activity) + return "Done" + + +@app.route("/migration1_step2") +@login_required +def tmp_migrate2(): + for activity in DB.activities.find(): + if ( + activity["box"] == Box.OUTBOX.value + and activity["type"] == ActivityType.LIKE.value + ): + like = ap.parse_activity(activity["activity"]) + obj = like.get_object() + DB.activities.update_one( + {"remote_id": like.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + elif activity["type"] == ActivityType.ANNOUNCE.value: + announce = ap.parse_activity(activity["activity"]) + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + return "Done" + + @app.route("/") def index(): if is_api_request(): @@ -437,89 +492,44 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - "type": "Create", - "activity.object.type": "Note", + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "activity.object.inReplyTo": None, "meta.deleted": False, + "meta.undo": False, } c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit - ).sort("_id", -1) - ) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - for data in outbox_data: - if data["type"] == "Announce": - if data["activity"]["object"].startswith("http"): - data["ref"] = { - "activity": { - "object": OBJECT_SERVICE.get(data["activity"]["object"]), - "id": "NA", - }, - "meta": {}, - } - print(data) - - - return render_template( - "index.html", - me=ME, - notes=DB.inbox.find( - {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - ).count(), - followers=DB.followers.count(), - following=DB.following.count(), - outbox_data=outbox_data, - cursor=cursor, - ) + return render_template("index.html", outbox_data=outbox_data, cursor=cursor) @app.route("/with_replies") def with_replies(): + # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 - q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + q = { + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + "meta.undo": False, + } c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit - ).sort("_id", -1) - ) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - for data in outbox_data: - if data["type"] == "Announce": - print(data) - if data["activity"]["object"].startswith("http"): - data["ref"] = { - "activity": { - "object": OBJECT_SERVICE.get(data["activity"]["object"]) - }, - "meta": {}, - } - - return render_template( - "index.html", - me=ME, - notes=DB.inbox.find( - {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - ).count(), - followers=DB.followers.count(), - following=DB.following.count(), - outbox_data=outbox_data, - cursor=cursor, - ) + return render_template("index.html", outbox_data=outbox_data, cursor=cursor) def _build_thread(data, include_children=True): @@ -534,12 +544,7 @@ def _build_thread(data, include_children=True): ) # Fetch the root replies, and the children - replies = ( - [data] - + list(DB.inbox.find(query)) - + list(DB.outbox.find(query)) - + list(DB.threads.find(query)) - ) + replies = [data] + list(DB.activities.find(query)) replies = sorted(replies, key=lambda d: d["activity"]["object"]["published"]) # Index all the IDs in order to build a tree idx = {} @@ -580,7 +585,9 @@ def _build_thread(data, include_children=True): @app.route("/note/") def note_by_id(note_id): - data = DB.outbox.find_one({"remote_id": back.activity_url(note_id)}) + data = DB.activities.find_one( + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(note_id)} + ) if not data: abort(404) if data["meta"].get("deleted", False): @@ -588,7 +595,7 @@ def note_by_id(note_id): thread = _build_thread(data) likes = list( - DB.inbox.find( + DB.activities.find( { "meta.undo": False, "type": ActivityType.LIKE.value, @@ -602,7 +609,7 @@ def note_by_id(note_id): likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] shares = list( - DB.inbox.find( + DB.activities.find( { "meta.undo": False, "type": ActivityType.ANNOUNCE.value, @@ -616,7 +623,7 @@ def note_by_id(note_id): shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] return render_template( - "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data + "note.html", likes=likes, shares=shares, thread=thread, note=data ) @@ -636,7 +643,10 @@ def nodeinfo(): "protocols": ["activitypub"], "services": {"inbound": [], "outbound": []}, "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "usage": { + "users": {"total": 1}, + "localPosts": DB.activities.count({"box": Box.OUTBOX.value}), + }, "metadata": { "sourceCode": "https://github.com/tsileo/microblog.pub", "nodeName": f"@{USERNAME}@{DOMAIN}", @@ -734,12 +744,13 @@ def outbox(): # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { + "box": Box.OUTBOX.value, "meta.deleted": False, # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: activity_from_doc(doc, embed=True), @@ -765,7 +776,9 @@ def outbox(): @app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({"remote_id": back.activity_url(item_id)}) + doc = DB.activities.find_one( + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} + ) if doc["meta"].get("deleted", False): obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) @@ -777,8 +790,12 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -793,8 +810,12 @@ def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -810,7 +831,7 @@ def outbox_activity_replies(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"], @@ -825,8 +846,12 @@ def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -845,7 +870,7 @@ def outbox_activity_likes(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), @@ -860,8 +885,12 @@ def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -880,7 +909,7 @@ def outbox_activity_shares(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), @@ -893,16 +922,21 @@ def outbox_activity_shares(item_id): @app.route("/admin", methods=["GET"]) @login_required def admin(): - q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} - col_liked = DB.outbox.count(q) + q = { + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.LIKE.value, + "box": Box.OUTBOX.value, + } + col_liked = DB.activities.count(q) return render_template( "admin.html", instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), + inbox_size=DB.activities.count({"box": Box.INBOX.value}), + outbox_size=DB.activities.count({"box": Box.OUTBOX.value}), + object_cache_size=0, + actor_cache_size=0, col_liked=col_liked, col_followers=DB.followers.count(), col_following=DB.following.count(), @@ -916,11 +950,9 @@ def new(): content = "" thread = [] if request.args.get("reply"): - data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) + data = DB.activities.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) - if not data: - abort(400) + abort(400) reply = ap.parse_activity(data["activity"]) reply_id = reply.id @@ -930,7 +962,7 @@ def new(): domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants content = f"@{actor.preferredUsername}@{domain} " - thread = _build_thread(data, include_children=False) + thread = _build_thread(data) return render_template("new.html", reply=reply_id, content=content, thread=thread) @@ -940,43 +972,42 @@ def new(): def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 - q = { - "type": "Create", + # FIXME(tsileo): show unfollow (performed by the current actor) and liked??? + mentions_query = { + "type": ActivityType.CREATE.value, "activity.object.tag.type": "Mention", "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", "meta.deleted": False, } - # TODO(tsileo): also include replies via regex on Create replyTo - q = { - "$or": [ - q, - {"type": "Follow"}, - {"type": "Accept"}, - {"type": "Undo", "activity.object.type": "Follow"}, - {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, - {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, - ] + replies_query = { + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}, + } + announced_query = { + "type": ActivityType.ANNOUNCE.value, + "activity.object": {"$regex": f"^{BASE_URL}"}, + } + new_followers_query = {"type": ActivityType.FOLLOW.value} + followed_query = {"type": ActivityType.ACCEPT.value} + q = { + "box": Box.INBOX.value, + "$or": [ + mentions_query, + announced_query, + replies_query, + new_followers_query, + followed_query, + ], } - print(q) c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - # TODO(tsileo): fix the annonce handling, copy it from /stream - # for data in outbox_data: - # if data['type'] == 'Announce': - # print(data) - # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - # data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} - # out.append(data) - # else: - # out.append(data) - return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) @@ -1061,8 +1092,11 @@ def api_like(): @api_required def api_undo(): oid = _user_api_arg("id") - doc = DB.outbox.find_one( - {"$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}]} + doc = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}], + } ) if not doc: raise ActivityNotFoundError(f"cannot found {oid}") @@ -1080,50 +1114,24 @@ def api_undo(): def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 + c = request.args.get("cursor") q = { - "type": "Create", - "activity.object.type": "Note", - "activity.object.inReplyTo": None, + "box": Box.INBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, } - c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( - "activity.published", -1 - ) + inbox_data = list( + # FIXME(tsileo): reshape using meta.cached_object + DB.activities.find(q, limit=limit).sort("_id", -1) ) cursor = None - if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]["_id"]) + if inbox_data and len(inbox_data) == limit: + cursor = str(inbox_data[-1]["_id"]) - out = [] - objcache = {} - cached = list( - DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( - "meta.announce_published", -1 - ) - ) - for c in cached: - objcache[c["object_id"]] = c["cached_object"] - for data in outbox_data: - if data["type"] == "Announce": - if ( - data["activity"]["object"].startswith("http") - and data["activity"]["object"] in objcache - ): - data["ref"] = { - "activity": {"object": objcache[data["activity"]["object"]]}, - "meta": {}, - } - out.append(data) - else: - print("OMG", data) - else: - out.append(data) - return render_template("stream.html", inbox_data=out, cursor=cursor) + return render_template("stream.html", inbox_data=inbox_data, cursor=cursor) @app.route("/inbox", methods=["GET", "POST"]) @@ -1138,8 +1146,8 @@ def inbox(): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, - q={"meta.deleted": False}, + DB.activities, + q={"meta.deleted": False, "box": Box.INBOX.value}, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), ) @@ -1198,9 +1206,9 @@ def api_debug(): return flask_jsonify(message="DB dropped") return flask_jsonify( - inbox=DB.inbox.count(), - outbox=DB.outbox.count(), - outbox_data=without_id(DB.outbox.find()), + inbox=DB.activities.count({"box": Box.INBOX.value}), + outbox=DB.activities.count({"box": Box.OUTBOX.value}), + outbox_data=without_id(DB.activities.find({"box": Box.OUTBOX.value})), ) @@ -1305,8 +1313,13 @@ def api_stream(): def api_block(): actor = _user_api_arg("actor") - existing = DB.outbox.find_one( - {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + existing = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "type": ActivityType.BLOCK.value, + "activity.object": actor, + "meta.undo": False, + } ) if existing: return _user_api_response(activity=existing["activity"]["id"]) @@ -1346,14 +1359,7 @@ def followers(): followers = [ ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) ] - return render_template( - "followers.html", - me=ME, - notes=DB.inbox.find({"object.object.type": "Note"}).count(), - followers=DB.followers.count(), - following=DB.following.count(), - followers_data=followers, - ) + return render_template("followers.html", followers_data=followers) @app.route("/following") @@ -1370,30 +1376,27 @@ def following(): following = [ ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) ] - return render_template( - "following.html", - me=ME, - notes=DB.inbox.find({"object.object.type": "Note"}).count(), - followers=DB.followers.count(), - following=DB.following.count(), - following_data=following, - ) + return render_template("following.html", following_data=following) @app.route("/tags/") def tags(tag): - if not DB.outbox.count( - {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + if not DB.activities.count( + { + "box": Box.OUTBOX.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } ): abort(404) if not is_api_request(): return render_template( "tags.html", tag=tag, - outbox_data=DB.outbox.find( + outbox_data=DB.activities.find( { - "type": "Create", - "activity.object.type": "Note", + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, "meta.deleted": False, "activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag, @@ -1401,6 +1404,7 @@ def tags(tag): ), ) q = { + "box": Box.OUTBOX.value, "meta.deleted": False, "meta.undo": False, "type": ActivityType.CREATE.value, @@ -1409,7 +1413,7 @@ def tags(tag): } return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"]["id"], @@ -1423,9 +1427,9 @@ def liked(): if not is_api_request(): return render_template( "liked.html", - me=ME, - liked=DB.outbox.find( + liked=DB.activities.find( { + "box": Box.OUTBOX.value, "type": ActivityType.LIKE.value, "meta.deleted": False, "meta.undo": False, @@ -1436,7 +1440,7 @@ def liked(): q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"], diff --git a/templates/index.html b/templates/index.html index 09ffb94..285e306 100644 --- a/templates/index.html +++ b/templates/index.html @@ -23,16 +23,15 @@
{% for item in outbox_data %} - {% if item.type == 'Announce' %} + {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %}

{{ boost_actor.name }} boosted

- {% if item.ref %} - {{ utils.display_note(item.ref, ui=False) }} - {% endif %} - - {% elif item.type == 'Create' %} - {{ utils.display_note(item) }} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object, ui=False) }} {% endif %} + {% elif item | has_type('Create') %} + {{ utils.display_note(item.activity.object, meta=item.meta) }} + {% endif %} {% endfor %}
diff --git a/templates/liked.html b/templates/liked.html new file mode 100644 index 0000000..b90462d --- /dev/null +++ b/templates/liked.html @@ -0,0 +1,18 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+ + +{% include "header.html" %} + +
+ {% for item in liked %} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object) }} + {% endif %} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/stream.html b/templates/stream.html index 9a0cff7..994e3d5 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -8,26 +8,32 @@
{% for item in inbox_data %} - {% if item.type == 'Create' %} - {{ utils.display_note(item, ui=True) }} + {% if item | has_type('Create') %} + {{ utils.display_note(item.activity.object, ui=True) }} {% else %} - {% if item.type == 'Announce' %} - + {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %}

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

- {% if item.ref %} - {{ utils.display_note(item.ref, ui=True) }} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} {% endif %} - {% if item.type == 'Follow' %} -

{{ item.activity.actor }} followed you

- {% elif item.type == 'Accept' %} -

you followed {{ item.activity.actor }}

- {% elif item.type == 'Undo' %} -

{{ item.activity.actor }} unfollowed you

+ {% if item | has_type('Follow') %} +

new follower +

+ {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} +
+ + {% elif item | has_type('Accept') %} +

you started following

+
+ {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} +
+ {% else %} + {% endif %} diff --git a/templates/tags.html b/templates/tags.html index 0df993c..3beaa8b 100644 --- a/templates/tags.html +++ b/templates/tags.html @@ -23,7 +23,7 @@

#{{ tag }}

{% for item in outbox_data %} - {{ utils.display_note(item) }} + {{ utils.display_note(item.activity.object, meta=item.meta) }} {% endfor %}
diff --git a/templates/utils.html b/templates/utils.html index 4314fbd..9fd10c2 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -14,27 +14,10 @@ {%- endmacro %} -{% macro display_actor(follower) -%} - -
-
-{% if not follower.icon %} - -{% else %} -{% endif %} -
-
-

{{ follower.name or follower.preferredUsername }}

-@{{ follower.preferredUsername }}@{{ follower.url | domain }} -
{{ follower.summary | safe }}
-
-
- -{%- endmacro %} -{% macro display_note(item, perma=False, ui=False, likes=[], shares=[]) -%} -{% set actor = item.activity.object.attributedTo | get_actor %} -
+{% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}) -%} +{% set actor = obj.attributedTo | get_actor %} +
@@ -47,78 +30,79 @@ {% if not perma %} - - + + {% endif %} - {% if item.activity.object.summary %}

{{ item.activity.object.summary }}

{% endif %} + {% if obj.summary %}

{{ obj.summary | clean }}

{% endif %}
- {{ item.activity.object.content | safe }} + {{ obj.content | clean | safe }}
- {% if item.activity.object.attachment %} + {% if obj.attachment %}
- {% if item.activity.object.attachment | not_only_imgs %} + {% if obj.attachment | not_only_imgs %}

Attachment

{% endif %}
{% endif %}
-{% if perma %}{{ item.activity.object.published | format_time }} +{% if perma %}{{ obj.published | format_time }} {% else %} -permalink +permalink -{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} -{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} -{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% if meta.count_reply %}{{ meta.count_reply }} replies{% endif %} +{% if meta.count_boost %}{{ meta.count_boost }} boosts{% endif %} +{% if meta.count_like %}{{ meta.count_like }} likes{% endif %} {% endif %} {% if ui and session.logged_in %} -{% set aid = item.activity.object.id | quote_plus %} +{% set aid = obj.id | quote_plus %} reply -{% set redir = request.path + "#activity-" + item['_id'].__str__() %} +{% set perma_id = obj.id | permalink_id %} +{% set redir = request.path + "#activity-" + perma_id %} -{% if item.meta.boosted %} +{% if meta.boosted %}
- +
{% else %}
- +
{% endif %} -{% if item.meta.liked %} +{% if meta.liked %}
- +
{% else %}
- +
@@ -127,17 +111,17 @@ {% endif %} {% if session.logged_in %} -{% if item.activity.id | is_from_outbox %} +{% if obj.id | is_from_outbox %}
- +
{% else %}
- +
@@ -151,14 +135,14 @@
{% if likes %}
-

{{ item.meta.count_like }} likes

{% for like in likes %} +

{{ meta.count_like }} likes

{% for like in likes %} {{ display_actor_inline(like) }} {% endfor %}
{% endif %} {% if shares %}
-

{{ item.meta.count_boost }} boosts

{% for boost in shares %} +

{{ meta.count_boost }} boosts

{% for boost in shares %} {{ display_actor_inline(boost) }} {% endfor %}
@@ -177,9 +161,9 @@ {% macro display_thread(thread, likes=[], shares=[]) -%} {% for reply in thread %} {% if reply._requested %} -{{ display_note(reply, perma=True, ui=False, likes=likes, shares=shares) }} +{{ display_note(reply.activity.object, perma=True, ui=False, likes=likes, shares=shares, meta=reply.meta) }} {% else %} -{{ display_note(reply, perma=False, ui=True) }} +{{ display_note(reply.activity.object, perma=False, ui=True, meta=reply.meta) }} {% endif %} {% endfor %} {% endmacro -%}